2024-11-13 07:33:19,978 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 07:33:19,992 main DEBUG Took 0.011950 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 07:33:19,993 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 07:33:19,993 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 07:33:19,994 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 07:33:19,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,002 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 07:33:20,014 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,017 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,018 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,019 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,020 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,020 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,021 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,023 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,023 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:33:20,024 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,024 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 07:33:20,026 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:33:20,027 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 07:33:20,028 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 07:33:20,029 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 07:33:20,030 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 07:33:20,030 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 07:33:20,038 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 07:33:20,040 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 07:33:20,042 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 07:33:20,042 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 07:33:20,043 main DEBUG createAppenders(={Console}) 2024-11-13 07:33:20,043 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-13 07:33:20,044 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 07:33:20,044 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-13 07:33:20,044 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 07:33:20,045 main DEBUG OutputStream closed 2024-11-13 07:33:20,045 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 07:33:20,045 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 07:33:20,046 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-13 07:33:20,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 07:33:20,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 07:33:20,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 07:33:20,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 07:33:20,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 07:33:20,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 07:33:20,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 07:33:20,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 07:33:20,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 07:33:20,118 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 07:33:20,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 07:33:20,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 07:33:20,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 07:33:20,119 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 07:33:20,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 07:33:20,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 07:33:20,120 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 07:33:20,121 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 07:33:20,123 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 07:33:20,124 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-13 07:33:20,124 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 07:33:20,125 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-13T07:33:20,333 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399 2024-11-13 07:33:20,335 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 07:33:20,336 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T07:33:20,344 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T07:33:20,350 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-13T07:33:20,371 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T07:33:20,411 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T07:33:20,412 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T07:33:20,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T07:33:20,437 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775, deleteOnExit=true 2024-11-13T07:33:20,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T07:33:20,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/test.cache.data in system properties and HBase conf 2024-11-13T07:33:20,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T07:33:20,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir in system properties and HBase conf 2024-11-13T07:33:20,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T07:33:20,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T07:33:20,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T07:33:20,518 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T07:33:20,612 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T07:33:20,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T07:33:20,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T07:33:20,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T07:33:20,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T07:33:20,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T07:33:20,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T07:33:20,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T07:33:20,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T07:33:20,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T07:33:20,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/nfs.dump.dir in system properties and HBase conf 2024-11-13T07:33:20,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/java.io.tmpdir in system properties and HBase conf 2024-11-13T07:33:20,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T07:33:20,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T07:33:20,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T07:33:21,701 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T07:33:21,769 INFO [Time-limited test {}] log.Log(170): Logging initialized @2458ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T07:33:21,834 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:33:21,890 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:33:21,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:33:21,910 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:33:21,911 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:33:21,930 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:33:21,934 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:33:21,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:33:22,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/java.io.tmpdir/jetty-localhost-42927-hadoop-hdfs-3_4_1-tests_jar-_-any-6793859714334865808/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T07:33:22,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:42927} 2024-11-13T07:33:22,144 INFO [Time-limited test {}] server.Server(415): Started @2834ms 2024-11-13T07:33:22,666 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:33:22,675 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:33:22,676 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:33:22,676 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:33:22,676 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T07:33:22,677 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:33:22,678 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:33:22,785 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/java.io.tmpdir/jetty-localhost-43173-hadoop-hdfs-3_4_1-tests_jar-_-any-8314582270649318737/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:33:22,786 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:43173} 2024-11-13T07:33:22,786 INFO [Time-limited test {}] server.Server(415): Started @3475ms 2024-11-13T07:33:22,838 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:33:22,954 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:33:22,961 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:33:22,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:33:22,965 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:33:22,965 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:33:22,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:33:22,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:33:23,071 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/java.io.tmpdir/jetty-localhost-35847-hadoop-hdfs-3_4_1-tests_jar-_-any-6603560478621863899/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:33:23,071 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:35847} 2024-11-13T07:33:23,071 INFO [Time-limited test {}] server.Server(415): Started @3761ms 2024-11-13T07:33:23,073 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:33:23,134 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:33:23,141 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:33:23,142 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:33:23,143 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:33:23,143 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:33:23,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:33:23,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:33:23,241 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/java.io.tmpdir/jetty-localhost-38011-hadoop-hdfs-3_4_1-tests_jar-_-any-5037084026829038731/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:33:23,242 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:38011} 2024-11-13T07:33:23,242 INFO [Time-limited test {}] server.Server(415): Started @3931ms 2024-11-13T07:33:23,244 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:33:24,464 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data3/current/BP-951039070-172.17.0.3-1731483201138/current, will proceed with Du for space computation calculation, 2024-11-13T07:33:24,464 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data4/current/BP-951039070-172.17.0.3-1731483201138/current, will proceed with Du for space computation calculation, 2024-11-13T07:33:24,464 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data2/current/BP-951039070-172.17.0.3-1731483201138/current, will proceed with Du for space computation calculation, 2024-11-13T07:33:24,464 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data1/current/BP-951039070-172.17.0.3-1731483201138/current, will proceed with Du for space computation calculation, 2024-11-13T07:33:24,492 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:33:24,492 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:33:24,537 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3c239933465e4c4 with lease ID 0xf8cb616be0896629: Processing first storage report for DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f from datanode DatanodeRegistration(127.0.0.1:41959, datanodeUuid=f073f10b-1213-44e7-916a-81d6e3bf0107, infoPort=34579, infoSecurePort=0, ipcPort=36663, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138) 2024-11-13T07:33:24,538 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3c239933465e4c4 with lease ID 0xf8cb616be0896629: from storage DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f node DatanodeRegistration(127.0.0.1:41959, datanodeUuid=f073f10b-1213-44e7-916a-81d6e3bf0107, infoPort=34579, infoSecurePort=0, ipcPort=36663, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T07:33:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfb11713a45983bed with lease ID 0xf8cb616be089662a: Processing first storage report for DS-dc23322e-2bc3-425b-94e5-86976802cb15 from datanode DatanodeRegistration(127.0.0.1:38619, datanodeUuid=f0ea07c0-a733-458f-ac7e-ccd7994cd7fd, infoPort=41899, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138) 2024-11-13T07:33:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfb11713a45983bed with lease ID 0xf8cb616be089662a: from storage DS-dc23322e-2bc3-425b-94e5-86976802cb15 node DatanodeRegistration(127.0.0.1:38619, datanodeUuid=f0ea07c0-a733-458f-ac7e-ccd7994cd7fd, infoPort=41899, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T07:33:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3c239933465e4c4 with lease ID 0xf8cb616be0896629: Processing first storage report for DS-1dff5cb3-530f-4834-ad8a-a5369ccccdda from datanode DatanodeRegistration(127.0.0.1:41959, datanodeUuid=f073f10b-1213-44e7-916a-81d6e3bf0107, infoPort=34579, infoSecurePort=0, ipcPort=36663, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138) 2024-11-13T07:33:24,539 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3c239933465e4c4 with lease ID 0xf8cb616be0896629: from storage DS-1dff5cb3-530f-4834-ad8a-a5369ccccdda node DatanodeRegistration(127.0.0.1:41959, datanodeUuid=f073f10b-1213-44e7-916a-81d6e3bf0107, infoPort=34579, infoSecurePort=0, ipcPort=36663, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:33:24,540 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfb11713a45983bed with lease ID 0xf8cb616be089662a: Processing first storage report for DS-c56f72c3-39aa-4499-84e9-a0406fa5f68c from datanode DatanodeRegistration(127.0.0.1:38619, datanodeUuid=f0ea07c0-a733-458f-ac7e-ccd7994cd7fd, infoPort=41899, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138) 2024-11-13T07:33:24,540 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfb11713a45983bed with lease ID 0xf8cb616be089662a: from storage DS-c56f72c3-39aa-4499-84e9-a0406fa5f68c node DatanodeRegistration(127.0.0.1:38619, datanodeUuid=f0ea07c0-a733-458f-ac7e-ccd7994cd7fd, infoPort=41899, infoSecurePort=0, ipcPort=40399, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:33:24,543 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data5/current/BP-951039070-172.17.0.3-1731483201138/current, will proceed with Du for space computation calculation, 2024-11-13T07:33:24,543 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data6/current/BP-951039070-172.17.0.3-1731483201138/current, will proceed with Du for space computation calculation, 2024-11-13T07:33:24,563 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:33:24,567 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8d2b8d7bffd72eeb with lease ID 0xf8cb616be089662b: Processing first storage report for DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862 from datanode DatanodeRegistration(127.0.0.1:35659, datanodeUuid=afbb5454-2714-4ec2-969e-e2935062c7ce, infoPort=34891, infoSecurePort=0, ipcPort=35495, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138) 2024-11-13T07:33:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8d2b8d7bffd72eeb with lease ID 0xf8cb616be089662b: from storage DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862 node DatanodeRegistration(127.0.0.1:35659, datanodeUuid=afbb5454-2714-4ec2-969e-e2935062c7ce, infoPort=34891, infoSecurePort=0, ipcPort=35495, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:33:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8d2b8d7bffd72eeb with lease ID 0xf8cb616be089662b: Processing first storage report for DS-50d77c77-631c-47a8-b7cc-dbb6f4673341 from datanode DatanodeRegistration(127.0.0.1:35659, datanodeUuid=afbb5454-2714-4ec2-969e-e2935062c7ce, infoPort=34891, infoSecurePort=0, ipcPort=35495, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138) 2024-11-13T07:33:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8d2b8d7bffd72eeb with lease ID 0xf8cb616be089662b: from storage DS-50d77c77-631c-47a8-b7cc-dbb6f4673341 node DatanodeRegistration(127.0.0.1:35659, datanodeUuid=afbb5454-2714-4ec2-969e-e2935062c7ce, infoPort=34891, infoSecurePort=0, ipcPort=35495, storageInfo=lv=-57;cid=testClusterID;nsid=661681744;c=1731483201138), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:33:24,665 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399 2024-11-13T07:33:24,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/zookeeper_0, clientPort=65209, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T07:33:24,735 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65209 2024-11-13T07:33:24,746 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:24,749 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:24,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:33:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:33:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:33:25,369 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b with version=8 2024-11-13T07:33:25,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/hbase-staging 2024-11-13T07:33:25,670 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:33:25,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:25,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:25,683 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:33:25,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:25,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:33:25,807 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T07:33:25,861 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T07:33:25,869 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T07:33:25,872 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:33:25,895 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 90298 (auto-detected) 2024-11-13T07:33:25,896 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T07:33:25,914 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44967 2024-11-13T07:33:25,933 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44967 connecting to ZooKeeper ensemble=127.0.0.1:65209 2024-11-13T07:33:26,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449670x0, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:33:26,065 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44967-0x101331cf8110000 connected 2024-11-13T07:33:26,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,179 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:33:26,196 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b, hbase.cluster.distributed=false 2024-11-13T07:33:26,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:33:26,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44967 2024-11-13T07:33:26,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44967 2024-11-13T07:33:26,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44967 2024-11-13T07:33:26,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44967 2024-11-13T07:33:26,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44967 2024-11-13T07:33:26,316 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:33:26,318 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,318 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,318 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:33:26,318 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:33:26,321 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:33:26,324 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:33:26,325 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39583 2024-11-13T07:33:26,327 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39583 connecting to ZooKeeper ensemble=127.0.0.1:65209 2024-11-13T07:33:26,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,331 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,348 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395830x0, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:33:26,349 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39583-0x101331cf8110001 connected 2024-11-13T07:33:26,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:33:26,353 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:33:26,360 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:33:26,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:33:26,368 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:33:26,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39583 2024-11-13T07:33:26,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39583 2024-11-13T07:33:26,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39583 2024-11-13T07:33:26,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39583 2024-11-13T07:33:26,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39583 2024-11-13T07:33:26,387 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:33:26,387 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,387 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,387 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:33:26,387 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:33:26,388 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:33:26,388 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:33:26,389 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37319 2024-11-13T07:33:26,391 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37319 connecting to ZooKeeper ensemble=127.0.0.1:65209 2024-11-13T07:33:26,391 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,393 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373190x0, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:33:26,407 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:373190x0, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:33:26,407 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37319-0x101331cf8110002 connected 2024-11-13T07:33:26,407 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:33:26,408 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:33:26,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:33:26,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:33:26,411 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37319 2024-11-13T07:33:26,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37319 2024-11-13T07:33:26,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37319 2024-11-13T07:33:26,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37319 2024-11-13T07:33:26,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37319 2024-11-13T07:33:26,429 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:33:26,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,429 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:33:26,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:33:26,430 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:33:26,430 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:33:26,430 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:33:26,431 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38607 2024-11-13T07:33:26,432 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38607 connecting to ZooKeeper ensemble=127.0.0.1:65209 2024-11-13T07:33:26,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,436 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386070x0, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:33:26,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386070x0, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:33:26,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38607-0x101331cf8110003 connected 2024-11-13T07:33:26,449 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:33:26,450 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:33:26,451 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:33:26,453 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:33:26,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38607 2024-11-13T07:33:26,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38607 2024-11-13T07:33:26,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38607 2024-11-13T07:33:26,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38607 2024-11-13T07:33:26,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38607 2024-11-13T07:33:26,471 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2998b885bdef:44967 2024-11-13T07:33:26,472 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2998b885bdef,44967,1731483205524 2024-11-13T07:33:26,485 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,487 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2998b885bdef,44967,1731483205524 2024-11-13T07:33:26,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:33:26,516 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:33:26,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:33:26,516 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,517 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T07:33:26,518 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2998b885bdef,44967,1731483205524 from backup master directory 2024-11-13T07:33:26,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2998b885bdef,44967,1731483205524 2024-11-13T07:33:26,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,532 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:33:26,533 WARN [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:33:26,534 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2998b885bdef,44967,1731483205524 2024-11-13T07:33:26,536 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T07:33:26,538 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T07:33:26,592 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/hbase.id] with ID: 24009036-1b1f-487a-9cff-0cd7f006815f 2024-11-13T07:33:26,592 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/.tmp/hbase.id 2024-11-13T07:33:26,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:33:26,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:33:26,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:33:26,610 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/.tmp/hbase.id]:[hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/hbase.id] 2024-11-13T07:33:26,658 INFO [master/2998b885bdef:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:33:26,664 INFO [master/2998b885bdef:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T07:33:26,683 INFO [master/2998b885bdef:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-13T07:33:26,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,743 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:26,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:33:26,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:33:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:33:26,781 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T07:33:26,783 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T07:33:26,788 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:26,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:33:26,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:33:26,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:33:26,836 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store 2024-11-13T07:33:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:33:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:33:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:33:26,861 INFO [master/2998b885bdef:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T07:33:26,864 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:26,866 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T07:33:26,866 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:33:26,866 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:33:26,867 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T07:33:26,868 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:33:26,868 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:33:26,869 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731483206865Disabling compacts and flushes for region at 1731483206865Disabling writes for close at 1731483206868 (+3 ms)Writing region close event to WAL at 1731483206868Closed at 1731483206868 2024-11-13T07:33:26,871 WARN [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/.initializing 2024-11-13T07:33:26,871 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/WALs/2998b885bdef,44967,1731483205524 2024-11-13T07:33:26,878 INFO [master/2998b885bdef:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:26,892 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C44967%2C1731483205524, suffix=, logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/WALs/2998b885bdef,44967,1731483205524, archiveDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/oldWALs, maxLogs=10 2024-11-13T07:33:26,919 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/WALs/2998b885bdef,44967,1731483205524/2998b885bdef%2C44967%2C1731483205524.1731483206896, exclude list is [], retry=0 2024-11-13T07:33:26,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:26,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:26,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:26,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:26,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T07:33:26,974 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-13T07:33:27,029 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/WALs/2998b885bdef,44967,1731483205524/2998b885bdef%2C44967%2C1731483205524.1731483206896 2024-11-13T07:33:27,030 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:27,031 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:27,031 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:27,034 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,034 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T07:33:27,090 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:27,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T07:33:27,097 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:27,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T07:33:27,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:27,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T07:33:27,105 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:27,107 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,110 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,111 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,116 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,116 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,120 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:33:27,124 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:33:27,128 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:27,129 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59107459, jitterRate=-0.1192302256822586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:33:27,136 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731483207045Initializing all the Stores at 1731483207047 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483207047Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483207048 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483207048Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483207048Cleaning up temporary data from old regions at 1731483207117 (+69 ms)Region opened successfully at 1731483207136 (+19 ms) 2024-11-13T07:33:27,137 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T07:33:27,169 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba9107a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:33:27,197 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T07:33:27,206 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T07:33:27,207 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T07:33:27,210 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T07:33:27,211 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T07:33:27,215 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T07:33:27,215 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T07:33:27,239 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T07:33:27,246 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T07:33:27,311 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T07:33:27,315 INFO [master/2998b885bdef:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T07:33:27,317 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T07:33:27,326 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T07:33:27,328 INFO [master/2998b885bdef:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T07:33:27,331 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T07:33:27,342 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T07:33:27,344 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T07:33:27,353 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T07:33:27,373 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T07:33:27,384 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T07:33:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:33:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:33:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:33:27,400 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:33:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,401 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,404 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2998b885bdef,44967,1731483205524, sessionid=0x101331cf8110000, setting cluster-up flag (Was=false) 2024-11-13T07:33:27,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,438 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,475 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T07:33:27,481 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2998b885bdef,44967,1731483205524 2024-11-13T07:33:27,505 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:27,537 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T07:33:27,540 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2998b885bdef,44967,1731483205524 2024-11-13T07:33:27,549 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T07:33:27,561 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(746): ClusterId : 24009036-1b1f-487a-9cff-0cd7f006815f 2024-11-13T07:33:27,561 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(746): ClusterId : 24009036-1b1f-487a-9cff-0cd7f006815f 2024-11-13T07:33:27,561 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(746): ClusterId : 24009036-1b1f-487a-9cff-0cd7f006815f 2024-11-13T07:33:27,563 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:33:27,563 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:33:27,563 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:33:27,586 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:33:27,586 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:33:27,586 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:33:27,586 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:33:27,586 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:33:27,586 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:33:27,607 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:33:27,607 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:33:27,607 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:33:27,607 DEBUG [RS:2;2998b885bdef:38607 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2397d8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:33:27,607 DEBUG [RS:1;2998b885bdef:37319 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b7c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:33:27,607 DEBUG [RS:0;2998b885bdef:39583 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b1f4f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:33:27,621 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2998b885bdef:38607 2024-11-13T07:33:27,623 INFO [RS:2;2998b885bdef:38607 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:33:27,623 INFO [RS:2;2998b885bdef:38607 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:33:27,623 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:33:27,624 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T07:33:27,625 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2998b885bdef:39583 2024-11-13T07:33:27,625 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2998b885bdef:37319 2024-11-13T07:33:27,626 INFO [RS:0;2998b885bdef:39583 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:33:27,626 INFO [RS:1;2998b885bdef:37319 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:33:27,626 INFO [RS:0;2998b885bdef:39583 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:33:27,626 INFO [RS:1;2998b885bdef:37319 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:33:27,626 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44967,1731483205524 with port=38607, startcode=1731483206428 2024-11-13T07:33:27,626 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:33:27,626 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:33:27,627 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44967,1731483205524 with port=37319, startcode=1731483206386 2024-11-13T07:33:27,627 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44967,1731483205524 with port=39583, startcode=1731483206286 2024-11-13T07:33:27,633 INFO [master/2998b885bdef:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T07:33:27,638 DEBUG [RS:1;2998b885bdef:37319 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:33:27,638 DEBUG [RS:2;2998b885bdef:38607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:33:27,638 DEBUG [RS:0;2998b885bdef:39583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:33:27,639 INFO [master/2998b885bdef:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T07:33:27,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2998b885bdef,44967,1731483205524 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T07:33:27,652 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:33:27,652 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:33:27,652 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:33:27,652 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:33:27,653 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2998b885bdef:0, corePoolSize=10, maxPoolSize=10 2024-11-13T07:33:27,653 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,653 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:33:27,653 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,655 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731483237654 2024-11-13T07:33:27,657 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T07:33:27,659 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T07:33:27,663 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T07:33:27,663 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T07:33:27,664 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T07:33:27,664 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T07:33:27,665 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:33:27,666 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T07:33:27,665 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,668 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T07:33:27,671 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T07:33:27,672 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52161, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:33:27,672 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34535, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:33:27,672 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T07:33:27,672 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42469, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:33:27,674 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,674 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T07:33:27,675 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T07:33:27,674 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T07:33:27,677 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731483207676,5,FailOnTimeoutGroup] 2024-11-13T07:33:27,677 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T07:33:27,678 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731483207677,5,FailOnTimeoutGroup] 2024-11-13T07:33:27,678 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,678 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T07:33:27,680 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,680 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,684 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,37319,1731483206386 2024-11-13T07:33:27,687 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,37319,1731483206386 2024-11-13T07:33:27,695 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b 2024-11-13T07:33:27,695 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35647 2024-11-13T07:33:27,695 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:33:27,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:33:27,706 DEBUG [RS:1;2998b885bdef:37319 {}] zookeeper.ZKUtil(111): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,37319,1731483206386 2024-11-13T07:33:27,706 WARN [RS:1;2998b885bdef:37319 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:33:27,706 INFO [RS:1;2998b885bdef:37319 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:27,706 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,37319,1731483206386 2024-11-13T07:33:27,708 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,37319,1731483206386] 2024-11-13T07:33:27,712 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,39583,1731483206286 2024-11-13T07:33:27,713 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,39583,1731483206286 2024-11-13T07:33:27,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:33:27,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:33:27,719 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b 2024-11-13T07:33:27,719 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35647 2024-11-13T07:33:27,719 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:33:27,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:33:27,721 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T07:33:27,722 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b 2024-11-13T07:33:27,727 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T07:33:27,727 WARN [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T07:33:27,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:33:27,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:33:27,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:33:27,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:27,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:33:27,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:33:27,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:27,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:33:27,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:33:27,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:27,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:33:27,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:33:27,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:27,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:33:27,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:33:27,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:27,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:27,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:33:27,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740 2024-11-13T07:33:27,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740 2024-11-13T07:33:27,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:33:27,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:33:27,761 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:33:27,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:33:27,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:33:27,765 DEBUG [RS:0;2998b885bdef:39583 {}] zookeeper.ZKUtil(111): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,39583,1731483206286 2024-11-13T07:33:27,765 WARN [RS:0;2998b885bdef:39583 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:33:27,765 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,39583,1731483206286] 2024-11-13T07:33:27,765 INFO [RS:0;2998b885bdef:39583 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:27,765 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286 2024-11-13T07:33:27,769 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:27,771 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62151037, jitterRate=-0.07387737929821014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:33:27,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731483207737Initializing all the Stores at 1731483207738 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483207739 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483207739Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483207739Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483207739Cleaning up temporary data from old regions at 1731483207760 (+21 ms)Region opened successfully at 1731483207776 (+16 ms) 2024-11-13T07:33:27,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:33:27,777 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:33:27,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:33:27,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:33:27,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:33:27,779 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:33:27,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731483207777Disabling compacts and flushes for region at 1731483207777Disabling writes for close at 1731483207777Writing region close event to WAL at 1731483207778 (+1 ms)Closed at 1731483207779 (+1 ms) 2024-11-13T07:33:27,783 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:33:27,784 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T07:33:27,784 INFO [RS:0;2998b885bdef:39583 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:33:27,784 INFO [RS:1;2998b885bdef:37319 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:33:27,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T07:33:27,799 INFO [RS:0;2998b885bdef:39583 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:33:27,799 INFO [RS:1;2998b885bdef:37319 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:33:27,803 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:33:27,806 INFO [RS:0;2998b885bdef:39583 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:33:27,806 INFO [RS:1;2998b885bdef:37319 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:33:27,806 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,806 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,808 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:33:27,808 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:33:27,808 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T07:33:27,814 INFO [RS:1;2998b885bdef:37319 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:33:27,815 INFO [RS:0;2998b885bdef:39583 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:33:27,816 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,816 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,816 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,816 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,816 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,816 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,816 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,816 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:33:27,817 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:33:27,817 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,817 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,818 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:33:27,818 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:33:27,818 DEBUG [RS:0;2998b885bdef:39583 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:33:27,818 DEBUG [RS:1;2998b885bdef:37319 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:33:27,819 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,819 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,819 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,819 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,819 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,819 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,39583,1731483206286-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:33:27,820 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,820 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,820 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,820 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,820 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,820 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,37319,1731483206386-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:33:27,828 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44967,1731483205524 with port=38607, startcode=1731483206428 2024-11-13T07:33:27,829 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,38607,1731483206428 2024-11-13T07:33:27,829 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,38607,1731483206428 2024-11-13T07:33:27,833 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b 2024-11-13T07:33:27,833 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35647 2024-11-13T07:33:27,833 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:33:27,844 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:33:27,844 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:33:27,846 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,37319,1731483206386-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,846 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,39583,1731483206286-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,846 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,846 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,846 INFO [RS:0;2998b885bdef:39583 {}] regionserver.Replication(171): 2998b885bdef,39583,1731483206286 started 2024-11-13T07:33:27,846 INFO [RS:1;2998b885bdef:37319 {}] regionserver.Replication(171): 2998b885bdef,37319,1731483206386 started 2024-11-13T07:33:27,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:33:27,853 DEBUG [RS:2;2998b885bdef:38607 {}] zookeeper.ZKUtil(111): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,38607,1731483206428 2024-11-13T07:33:27,854 WARN [RS:2;2998b885bdef:38607 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:33:27,854 INFO [RS:2;2998b885bdef:38607 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:27,854 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,38607,1731483206428] 2024-11-13T07:33:27,854 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428 2024-11-13T07:33:27,860 INFO [RS:2;2998b885bdef:38607 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:33:27,862 INFO [RS:2;2998b885bdef:38607 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:33:27,866 INFO [RS:2;2998b885bdef:38607 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:33:27,866 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,867 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:33:27,868 INFO [RS:2;2998b885bdef:38607 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:33:27,869 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,869 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,869 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,869 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,869 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,869 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,870 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:33:27,871 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:33:27,871 DEBUG [RS:2;2998b885bdef:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:33:27,871 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,871 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,871 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,39583,1731483206286, RpcServer on 2998b885bdef/172.17.0.3:39583, sessionid=0x101331cf8110001 2024-11-13T07:33:27,871 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,37319,1731483206386, RpcServer on 2998b885bdef/172.17.0.3:37319, sessionid=0x101331cf8110002 2024-11-13T07:33:27,872 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:33:27,872 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:33:27,872 DEBUG [RS:0;2998b885bdef:39583 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,39583,1731483206286 2024-11-13T07:33:27,872 DEBUG [RS:1;2998b885bdef:37319 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,37319,1731483206386 2024-11-13T07:33:27,872 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,39583,1731483206286' 2024-11-13T07:33:27,872 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,37319,1731483206386' 2024-11-13T07:33:27,872 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:33:27,872 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:33:27,873 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:33:27,873 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:33:27,874 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:33:27,874 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:33:27,874 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:33:27,874 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:33:27,874 DEBUG [RS:0;2998b885bdef:39583 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,39583,1731483206286 2024-11-13T07:33:27,874 DEBUG [RS:1;2998b885bdef:37319 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,37319,1731483206386 2024-11-13T07:33:27,874 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,39583,1731483206286' 2024-11-13T07:33:27,874 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,37319,1731483206386' 2024-11-13T07:33:27,874 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:33:27,874 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:33:27,875 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:33:27,875 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:33:27,875 DEBUG [RS:1;2998b885bdef:37319 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:33:27,875 DEBUG [RS:0;2998b885bdef:39583 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:33:27,875 INFO [RS:0;2998b885bdef:39583 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:33:27,875 INFO [RS:1;2998b885bdef:37319 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:33:27,875 INFO [RS:1;2998b885bdef:37319 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:33:27,875 INFO [RS:0;2998b885bdef:39583 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:33:27,876 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,876 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,876 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,876 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,876 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,876 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,38607,1731483206428-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:33:27,904 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:33:27,904 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,38607,1731483206428-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,904 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,904 INFO [RS:2;2998b885bdef:38607 {}] regionserver.Replication(171): 2998b885bdef,38607,1731483206428 started 2024-11-13T07:33:27,920 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:27,920 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,38607,1731483206428, RpcServer on 2998b885bdef/172.17.0.3:38607, sessionid=0x101331cf8110003 2024-11-13T07:33:27,920 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:33:27,920 DEBUG [RS:2;2998b885bdef:38607 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,38607,1731483206428 2024-11-13T07:33:27,920 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,38607,1731483206428' 2024-11-13T07:33:27,921 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:33:27,921 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:33:27,922 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:33:27,922 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:33:27,922 DEBUG [RS:2;2998b885bdef:38607 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,38607,1731483206428 2024-11-13T07:33:27,922 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,38607,1731483206428' 2024-11-13T07:33:27,922 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:33:27,923 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:33:27,924 DEBUG [RS:2;2998b885bdef:38607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:33:27,924 INFO [RS:2;2998b885bdef:38607 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:33:27,924 INFO [RS:2;2998b885bdef:38607 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:33:27,959 WARN [2998b885bdef:44967 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T07:33:27,983 INFO [RS:0;2998b885bdef:39583 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:27,984 INFO [RS:1;2998b885bdef:37319 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:27,987 INFO [RS:1;2998b885bdef:37319 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C37319%2C1731483206386, suffix=, logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,37319,1731483206386, archiveDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs, maxLogs=32 2024-11-13T07:33:27,987 INFO [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C39583%2C1731483206286, suffix=, logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286, archiveDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs, maxLogs=32 2024-11-13T07:33:28,004 DEBUG [RS:0;2998b885bdef:39583 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286/2998b885bdef%2C39583%2C1731483206286.1731483207992, exclude list is [], retry=0 2024-11-13T07:33:28,007 DEBUG [RS:1;2998b885bdef:37319 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,37319,1731483206386/2998b885bdef%2C37319%2C1731483206386.1731483207992, exclude list is [], retry=0 2024-11-13T07:33:28,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:28,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:28,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:28,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:28,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:28,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:28,025 INFO [RS:2;2998b885bdef:38607 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:28,028 INFO [RS:2;2998b885bdef:38607 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C38607%2C1731483206428, suffix=, logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428, archiveDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs, maxLogs=32 2024-11-13T07:33:28,043 INFO [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286/2998b885bdef%2C39583%2C1731483206286.1731483207992 2024-11-13T07:33:28,044 DEBUG [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:28,047 INFO [RS:1;2998b885bdef:37319 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,37319,1731483206386/2998b885bdef%2C37319%2C1731483206386.1731483207992 2024-11-13T07:33:28,047 DEBUG [RS:1;2998b885bdef:37319 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:28,048 DEBUG [RS:2;2998b885bdef:38607 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428/2998b885bdef%2C38607%2C1731483206428.1731483208030, exclude list is [], retry=0 2024-11-13T07:33:28,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:28,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:28,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:28,084 INFO [RS:2;2998b885bdef:38607 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428/2998b885bdef%2C38607%2C1731483206428.1731483208030 2024-11-13T07:33:28,084 DEBUG [RS:2;2998b885bdef:38607 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:33:28,213 DEBUG [2998b885bdef:44967 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T07:33:28,222 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T07:33:28,228 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:33:28,228 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:33:28,228 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T07:33:28,228 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:33:28,236 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:28,240 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,38607,1731483206428, state=OPENING 2024-11-13T07:33:28,290 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T07:33:28,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:28,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:28,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:28,300 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:33:28,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,303 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:33:28,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,38607,1731483206428}] 2024-11-13T07:33:28,487 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:33:28,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52083, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:33:28,499 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T07:33:28,500 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:28,501 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T07:33:28,504 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C38607%2C1731483206428.meta, suffix=.meta, logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428, archiveDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs, maxLogs=32 2024-11-13T07:33:28,518 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, exclude list is [], retry=0 2024-11-13T07:33:28,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:28,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:28,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:28,550 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta 2024-11-13T07:33:28,551 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:28,551 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:28,553 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T07:33:28,555 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T07:33:28,560 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T07:33:28,564 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T07:33:28,564 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:28,565 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T07:33:28,565 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T07:33:28,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:33:28,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:33:28,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:28,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:28,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:33:28,572 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:33:28,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:28,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:28,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:33:28,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:33:28,574 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:28,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:28,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:33:28,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:33:28,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:28,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:28,577 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:33:28,578 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740 2024-11-13T07:33:28,581 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740 2024-11-13T07:33:28,583 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:33:28,583 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:33:28,584 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:33:28,586 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:33:28,588 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69366246, jitterRate=0.03363761305809021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:33:28,588 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T07:33:28,589 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731483208565Writing region info on filesystem at 1731483208565Initializing all the Stores at 1731483208567 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483208568 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483208568Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483208568Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483208568Cleaning up temporary data from old regions at 1731483208583 (+15 ms)Running coprocessor post-open hooks at 1731483208588 (+5 ms)Region opened successfully at 1731483208589 (+1 ms) 2024-11-13T07:33:28,596 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731483208477 2024-11-13T07:33:28,607 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T07:33:28,607 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T07:33:28,609 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:28,612 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,38607,1731483206428, state=OPEN 2024-11-13T07:33:28,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:28,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:28,669 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:28,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:28,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:28,669 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2998b885bdef,38607,1731483206428 2024-11-13T07:33:28,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T07:33:28,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,38607,1731483206428 in 365 msec 2024-11-13T07:33:28,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T07:33:28,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 887 msec 2024-11-13T07:33:28,687 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:33:28,687 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T07:33:28,709 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:28,710 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1] 2024-11-13T07:33:28,729 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:33:28,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40035, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:33:28,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1690 sec 2024-11-13T07:33:28,753 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731483208752, completionTime=-1 2024-11-13T07:33:28,756 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T07:33:28,756 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T07:33:28,786 INFO [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T07:33:28,786 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731483268786 2024-11-13T07:33:28,787 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731483328787 2024-11-13T07:33:28,787 INFO [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-13T07:33:28,789 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:33:28,797 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44967,1731483205524-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,797 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44967,1731483205524-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,797 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44967,1731483205524-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,799 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2998b885bdef:44967, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,799 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,800 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,806 DEBUG [master/2998b885bdef:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T07:33:28,828 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.294sec 2024-11-13T07:33:28,830 INFO [master/2998b885bdef:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T07:33:28,831 INFO [master/2998b885bdef:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T07:33:28,832 INFO [master/2998b885bdef:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T07:33:28,833 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T07:33:28,833 INFO [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T07:33:28,834 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44967,1731483205524-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:33:28,834 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44967,1731483205524-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T07:33:28,839 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T07:33:28,840 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T07:33:28,841 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44967,1731483205524-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:28,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd494aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:33:28,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2998b885bdef,44967,-1 for getting cluster id 2024-11-13T07:33:28,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T07:33:28,887 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '24009036-1b1f-487a-9cff-0cd7f006815f' 2024-11-13T07:33:28,889 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T07:33:28,889 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "24009036-1b1f-487a-9cff-0cd7f006815f" 2024-11-13T07:33:28,890 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bd7bd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:33:28,890 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2998b885bdef,44967,-1] 2024-11-13T07:33:28,892 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T07:33:28,894 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:33:28,895 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T07:33:28,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75555b5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:33:28,898 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:28,905 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1] 2024-11-13T07:33:28,906 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:33:28,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:33:28,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2998b885bdef,44967,1731483205524 2024-11-13T07:33:28,934 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35647/hbase 2024-11-13T07:33:28,949 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=340, ProcessCount=11, AvailableMemoryMB=8898 2024-11-13T07:33:28,973 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:28,977 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:28,979 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:28,984 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61248580, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-61248580, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:29,006 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61248580/hregion-61248580.1731483208985, exclude list is [], retry=0 2024-11-13T07:33:29,010 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:29,010 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:29,010 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:29,040 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61248580/hregion-61248580.1731483208985 2024-11-13T07:33:29,041 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:29,041 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4b51c6af94870679f4d232c389e29290, NAME => 'testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741838_1014 (size=64) 2024-11-13T07:33:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741838_1014 (size=64) 2024-11-13T07:33:29,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741838_1014 (size=64) 2024-11-13T07:33:29,056 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:29,058 INFO [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,060 INFO [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b51c6af94870679f4d232c389e29290 columnFamilyName a 2024-11-13T07:33:29,060 DEBUG [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:29,061 INFO [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] regionserver.HStore(327): Store=4b51c6af94870679f4d232c389e29290/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:29,061 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,064 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,064 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,066 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,070 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:29,071 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b51c6af94870679f4d232c389e29290; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60978559, jitterRate=-0.09134866297245026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:29,072 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b51c6af94870679f4d232c389e29290: Writing region info on filesystem at 1731483209056Initializing all the Stores at 1731483209057 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483209057Cleaning up temporary data from old regions at 1731483209064 (+7 ms)Region opened successfully at 1731483209072 (+8 ms) 2024-11-13T07:33:29,072 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4b51c6af94870679f4d232c389e29290, disabling compactions & flushes 2024-11-13T07:33:29,072 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290. 2024-11-13T07:33:29,072 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290. 2024-11-13T07:33:29,072 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290. after waiting 0 ms 2024-11-13T07:33:29,072 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290. 2024-11-13T07:33:29,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290. 2024-11-13T07:33:29,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4b51c6af94870679f4d232c389e29290: Waiting for close lock at 1731483209072Disabling compacts and flushes for region at 1731483209072Disabling writes for close at 1731483209072Writing region close event to WAL at 1731483209073 (+1 ms)Closed at 1731483209073 2024-11-13T07:33:29,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741837_1013 (size=175) 2024-11-13T07:33:29,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741837_1013 (size=175) 2024-11-13T07:33:29,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741837_1013 (size=175) 2024-11-13T07:33:29,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:29,084 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61248580:(num 1731483208985) 2024-11-13T07:33:29,085 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T07:33:29,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741839_1015 (size=469) 2024-11-13T07:33:29,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741839_1015 (size=469) 2024-11-13T07:33:29,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741839_1015 (size=469) 2024-11-13T07:33:29,131 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T07:33:29,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741840_1016 (size=322) 2024-11-13T07:33:29,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741840_1016 (size=322) 2024-11-13T07:33:29,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741840_1016 (size=322) 2024-11-13T07:33:29,193 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1, size=469 (469bytes) 2024-11-13T07:33:29,194 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T07:33:29,194 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T07:33:29,194 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1 2024-11-13T07:33:29,199 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1 after 3ms 2024-11-13T07:33:29,234 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:29,235 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1 took 42ms 2024-11-13T07:33:29,240 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1 so closing down 2024-11-13T07:33:29,240 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:29,243 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T07:33:29,270 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T07:33:29,271 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:29,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741841_1017 (size=469) 2024-11-13T07:33:29,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741841_1017 (size=469) 2024-11-13T07:33:29,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741841_1017 (size=469) 2024-11-13T07:33:29,285 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:29,289 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002 2024-11-13T07:33:29,294 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T07:33:29,294 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1, journal: Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1, size=469 (469bytes) at 1731483209193Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1 so closing down at 1731483209240 (+47 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000001-wal-1.temp at 1731483209270 (+30 ms)3 split writer threads finished at 1731483209271 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731483209285 (+14 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002 at 1731483209289 (+4 ms)Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731483209294 (+5 ms) 2024-11-13T07:33:29,311 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2, size=322 (322bytes) 2024-11-13T07:33:29,311 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2 2024-11-13T07:33:29,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2 after 1ms 2024-11-13T07:33:29,343 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:29,343 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2 took 32ms 2024-11-13T07:33:29,346 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2 so closing down 2024-11-13T07:33:29,346 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:29,349 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T07:33:29,375 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T07:33:29,376 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:29,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741842_1018 (size=322) 2024-11-13T07:33:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741842_1018 (size=322) 2024-11-13T07:33:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741842_1018 (size=322) 2024-11-13T07:33:29,385 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:29,417 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:29,420 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-13T07:33:29,422 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T07:33:29,422 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2, journal: Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2, size=322 (322bytes) at 1731483209311Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2 so closing down at 1731483209346 (+35 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002-wal-2.temp at 1731483209375 (+29 ms)3 split writer threads finished at 1731483209376 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731483209385 (+9 ms)Processed 1 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731483209422 (+37 ms) 2024-11-13T07:33:29,422 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:29,426 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:29,440 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal.1731483209426, exclude list is [], retry=0 2024-11-13T07:33:29,444 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:29,445 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:29,445 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:29,472 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal.1731483209426 2024-11-13T07:33:29,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:29,473 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b51c6af94870679f4d232c389e29290, NAME => 'testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:29,473 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:29,473 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,473 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,475 INFO [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,477 INFO [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b51c6af94870679f4d232c389e29290 columnFamilyName a 2024-11-13T07:33:29,477 DEBUG [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:29,478 INFO [StoreOpener-4b51c6af94870679f4d232c389e29290-1 {}] regionserver.HStore(327): Store=4b51c6af94870679f4d232c389e29290/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:29,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,485 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002 2024-11-13T07:33:29,513 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:29,521 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002 2024-11-13T07:33:29,524 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b51c6af94870679f4d232c389e29290 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T07:33:29,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/.tmp/a/9059a73bdcd7420fbcd429b36485d4da is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731483209084/Put/seqid=0 2024-11-13T07:33:29,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741844_1020 (size=5170) 2024-11-13T07:33:29,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741844_1020 (size=5170) 2024-11-13T07:33:29,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741844_1020 (size=5170) 2024-11-13T07:33:29,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/.tmp/a/9059a73bdcd7420fbcd429b36485d4da 2024-11-13T07:33:29,669 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/.tmp/a/9059a73bdcd7420fbcd429b36485d4da as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/a/9059a73bdcd7420fbcd429b36485d4da 2024-11-13T07:33:29,679 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/a/9059a73bdcd7420fbcd429b36485d4da, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T07:33:29,686 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 4b51c6af94870679f4d232c389e29290 in 160ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T07:33:29,688 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/0000000000000000002 2024-11-13T07:33:29,688 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,688 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,692 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4b51c6af94870679f4d232c389e29290 2024-11-13T07:33:29,695 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/4b51c6af94870679f4d232c389e29290/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T07:33:29,697 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4b51c6af94870679f4d232c389e29290; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58918686, jitterRate=-0.12204316258430481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:29,698 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4b51c6af94870679f4d232c389e29290: Writing region info on filesystem at 1731483209474Initializing all the Stores at 1731483209475 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483209475Obtaining lock to block concurrent updates at 1731483209525 (+50 ms)Preparing flush snapshotting stores in 4b51c6af94870679f4d232c389e29290 at 1731483209525Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731483209528 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731483208975.4b51c6af94870679f4d232c389e29290. at 1731483209528Flushing 4b51c6af94870679f4d232c389e29290/a: creating writer at 1731483209529 (+1 ms)Flushing 4b51c6af94870679f4d232c389e29290/a: appending metadata at 1731483209597 (+68 ms)Flushing 4b51c6af94870679f4d232c389e29290/a: closing flushed file at 1731483209600 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3052174c: reopening flushed file at 1731483209667 (+67 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 4b51c6af94870679f4d232c389e29290 in 160ms, sequenceid=2, compaction requested=false; wal=null at 1731483209686 (+19 ms)Cleaning up temporary data from old regions at 1731483209689 (+3 ms)Region opened successfully at 1731483209698 (+9 ms) 2024-11-13T07:33:29,731 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:40610 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:45586 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35647/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:45568 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:40590 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:32976 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:33122 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 340), ProcessCount=11 (was 11), AvailableMemoryMB=8771 (was 8898) 2024-11-13T07:33:29,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=340, ProcessCount=11, AvailableMemoryMB=8767 2024-11-13T07:33:29,760 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:29,763 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:29,764 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:29,768 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-06710032, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-06710032, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:29,780 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-06710032/hregion-06710032.1731483209768, exclude list is [], retry=0 2024-11-13T07:33:29,784 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:29,784 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:29,784 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:29,810 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-06710032/hregion-06710032.1731483209768 2024-11-13T07:33:29,810 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:29,811 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d53e4c6b686ba4188e181db4769bc894, NAME => 'testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:29,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741846_1022 (size=64) 2024-11-13T07:33:29,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741846_1022 (size=64) 2024-11-13T07:33:29,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741846_1022 (size=64) 2024-11-13T07:33:29,823 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:29,826 INFO [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,828 INFO [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d53e4c6b686ba4188e181db4769bc894 columnFamilyName a 2024-11-13T07:33:29,828 DEBUG [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:29,829 INFO [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] regionserver.HStore(327): Store=d53e4c6b686ba4188e181db4769bc894/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:29,829 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,830 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,831 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,831 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,831 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,834 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:29,836 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:29,837 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d53e4c6b686ba4188e181db4769bc894; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73754665, jitterRate=0.09903015196323395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:29,837 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d53e4c6b686ba4188e181db4769bc894: Writing region info on filesystem at 1731483209823Initializing all the Stores at 1731483209825 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483209825Cleaning up temporary data from old regions at 1731483209831 (+6 ms)Region opened successfully at 1731483209837 (+6 ms) 2024-11-13T07:33:29,837 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d53e4c6b686ba4188e181db4769bc894, disabling compactions & flushes 2024-11-13T07:33:29,837 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894. 2024-11-13T07:33:29,837 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894. 2024-11-13T07:33:29,837 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894. after waiting 0 ms 2024-11-13T07:33:29,837 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894. 2024-11-13T07:33:29,838 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894. 2024-11-13T07:33:29,838 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d53e4c6b686ba4188e181db4769bc894: Waiting for close lock at 1731483209837Disabling compacts and flushes for region at 1731483209837Disabling writes for close at 1731483209837Writing region close event to WAL at 1731483209838 (+1 ms)Closed at 1731483209838 2024-11-13T07:33:29,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741845_1021 (size=175) 2024-11-13T07:33:29,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741845_1021 (size=175) 2024-11-13T07:33:29,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741845_1021 (size=175) 2024-11-13T07:33:29,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:29,846 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-06710032:(num 1731483209768) 2024-11-13T07:33:29,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T07:33:29,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741847_1023 (size=469) 2024-11-13T07:33:29,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741847_1023 (size=469) 2024-11-13T07:33:29,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741847_1023 (size=469) 2024-11-13T07:33:29,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T07:33:29,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741848_1024 (size=322) 2024-11-13T07:33:29,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741848_1024 (size=322) 2024-11-13T07:33:29,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741848_1024 (size=322) 2024-11-13T07:33:29,953 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2, size=322 (322bytes) 2024-11-13T07:33:29,953 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2 2024-11-13T07:33:29,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2 after 1ms 2024-11-13T07:33:29,981 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:29,981 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2 took 28ms 2024-11-13T07:33:29,983 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2 so closing down 2024-11-13T07:33:29,983 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:29,986 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T07:33:30,010 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T07:33:30,010 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:30,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741849_1025 (size=322) 2024-11-13T07:33:30,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741849_1025 (size=322) 2024-11-13T07:33:30,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741849_1025 (size=322) 2024-11-13T07:33:30,018 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:30,020 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 2024-11-13T07:33:30,021 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T07:33:30,021 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2, journal: Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2, size=322 (322bytes) at 1731483209953Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2 so closing down at 1731483209983 (+30 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002-wal-2.temp at 1731483210010 (+27 ms)3 split writer threads finished at 1731483210010Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731483210018 (+8 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 at 1731483210020 (+2 ms)Processed 1 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731483210021 (+1 ms) 2024-11-13T07:33:30,037 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1, size=469 (469bytes) 2024-11-13T07:33:30,037 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1 2024-11-13T07:33:30,038 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1 after 0ms 2024-11-13T07:33:30,065 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:30,065 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1 took 28ms 2024-11-13T07:33:30,068 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1 so closing down 2024-11-13T07:33:30,068 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:30,070 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T07:33:30,095 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T07:33:30,096 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:30,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741850_1026 (size=469) 2024-11-13T07:33:30,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741850_1026 (size=469) 2024-11-13T07:33:30,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741850_1026 (size=469) 2024-11-13T07:33:30,105 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:30,134 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:30,137 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002, length=322 2024-11-13T07:33:30,139 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 2024-11-13T07:33:30,139 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T07:33:30,139 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1, journal: Splitting hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1, size=469 (469bytes) at 1731483210037Finishing writing output for hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1 so closing down at 1731483210068 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000001-wal-1.temp at 1731483210095 (+27 ms)3 split writer threads finished at 1731483210096 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731483210105 (+9 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 at 1731483210139 (+34 ms)Processed 2 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731483210139 2024-11-13T07:33:30,139 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:30,142 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:30,154 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal.1731483210142, exclude list is [], retry=0 2024-11-13T07:33:30,157 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:30,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:30,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:30,186 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal.1731483210142 2024-11-13T07:33:30,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:30,187 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d53e4c6b686ba4188e181db4769bc894, NAME => 'testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:30,187 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:30,187 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,187 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,189 INFO [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,190 INFO [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d53e4c6b686ba4188e181db4769bc894 columnFamilyName a 2024-11-13T07:33:30,191 DEBUG [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:30,191 INFO [StoreOpener-d53e4c6b686ba4188e181db4769bc894-1 {}] regionserver.HStore(327): Store=d53e4c6b686ba4188e181db4769bc894/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:30,192 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,192 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,195 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 2024-11-13T07:33:30,221 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:30,223 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 2024-11-13T07:33:30,223 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d53e4c6b686ba4188e181db4769bc894 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T07:33:30,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/.tmp/a/bbe8b554b6ad4a6f90f822bf2ddc981d is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731483209846/Put/seqid=0 2024-11-13T07:33:30,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741852_1028 (size=5170) 2024-11-13T07:33:30,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741852_1028 (size=5170) 2024-11-13T07:33:30,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741852_1028 (size=5170) 2024-11-13T07:33:30,247 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/.tmp/a/bbe8b554b6ad4a6f90f822bf2ddc981d 2024-11-13T07:33:30,256 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/.tmp/a/bbe8b554b6ad4a6f90f822bf2ddc981d as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/a/bbe8b554b6ad4a6f90f822bf2ddc981d 2024-11-13T07:33:30,265 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/a/bbe8b554b6ad4a6f90f822bf2ddc981d, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T07:33:30,265 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d53e4c6b686ba4188e181db4769bc894 in 42ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T07:33:30,266 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/0000000000000000002 2024-11-13T07:33:30,266 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,267 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,269 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d53e4c6b686ba4188e181db4769bc894 2024-11-13T07:33:30,272 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/d53e4c6b686ba4188e181db4769bc894/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T07:33:30,273 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d53e4c6b686ba4188e181db4769bc894; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67769497, jitterRate=0.009844198822975159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:30,274 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d53e4c6b686ba4188e181db4769bc894: Writing region info on filesystem at 1731483210187Initializing all the Stores at 1731483210188 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483210188Obtaining lock to block concurrent updates at 1731483210223 (+35 ms)Preparing flush snapshotting stores in d53e4c6b686ba4188e181db4769bc894 at 1731483210223Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731483210223Flushing stores of testReplayEditsWrittenIntoWAL,,1731483209761.d53e4c6b686ba4188e181db4769bc894. at 1731483210223Flushing d53e4c6b686ba4188e181db4769bc894/a: creating writer at 1731483210223Flushing d53e4c6b686ba4188e181db4769bc894/a: appending metadata at 1731483210237 (+14 ms)Flushing d53e4c6b686ba4188e181db4769bc894/a: closing flushed file at 1731483210237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c4097c1: reopening flushed file at 1731483210254 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d53e4c6b686ba4188e181db4769bc894 in 42ms, sequenceid=2, compaction requested=false; wal=null at 1731483210265 (+11 ms)Cleaning up temporary data from old regions at 1731483210267 (+2 ms)Region opened successfully at 1731483210274 (+7 ms) 2024-11-13T07:33:30,290 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:45682 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:40610 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:33192 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:45568 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:32976 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:40672 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 340), ProcessCount=11 (was 11), AvailableMemoryMB=8760 (was 8767) 2024-11-13T07:33:30,300 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=337, ProcessCount=11, AvailableMemoryMB=8760 2024-11-13T07:33:30,316 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:30,318 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:30,318 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:30,322 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31534528, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-31534528, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:30,335 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31534528/hregion-31534528.1731483210322, exclude list is [], retry=0 2024-11-13T07:33:30,339 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:30,340 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:30,340 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:30,367 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31534528/hregion-31534528.1731483210322 2024-11-13T07:33:30,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:30,368 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 85a53355305906a9a234d60b26d36ae3, NAME => 'testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:30,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741854_1030 (size=64) 2024-11-13T07:33:30,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741854_1030 (size=64) 2024-11-13T07:33:30,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741854_1030 (size=64) 2024-11-13T07:33:30,379 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:30,381 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,383 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a53355305906a9a234d60b26d36ae3 columnFamilyName a 2024-11-13T07:33:30,383 DEBUG [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:30,384 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(327): Store=85a53355305906a9a234d60b26d36ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:30,384 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,386 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a53355305906a9a234d60b26d36ae3 columnFamilyName b 2024-11-13T07:33:30,386 DEBUG [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:30,387 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(327): Store=85a53355305906a9a234d60b26d36ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:30,387 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,388 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a53355305906a9a234d60b26d36ae3 columnFamilyName c 2024-11-13T07:33:30,389 DEBUG [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:30,389 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(327): Store=85a53355305906a9a234d60b26d36ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:30,389 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,392 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,392 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,393 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:30,394 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:30,397 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:30,397 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 85a53355305906a9a234d60b26d36ae3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73754423, jitterRate=0.09902654588222504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:30,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 85a53355305906a9a234d60b26d36ae3: Writing region info on filesystem at 1731483210379Initializing all the Stores at 1731483210380 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483210380Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483210381 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483210381Cleaning up temporary data from old regions at 1731483210392 (+11 ms)Region opened successfully at 1731483210397 (+5 ms) 2024-11-13T07:33:30,398 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 85a53355305906a9a234d60b26d36ae3, disabling compactions & flushes 2024-11-13T07:33:30,398 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:30,398 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:30,398 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. after waiting 0 ms 2024-11-13T07:33:30,398 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:30,399 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:30,399 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 85a53355305906a9a234d60b26d36ae3: Waiting for close lock at 1731483210398Disabling compacts and flushes for region at 1731483210398Disabling writes for close at 1731483210398Writing region close event to WAL at 1731483210399 (+1 ms)Closed at 1731483210399 2024-11-13T07:33:30,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741853_1029 (size=175) 2024-11-13T07:33:30,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741853_1029 (size=175) 2024-11-13T07:33:30,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741853_1029 (size=175) 2024-11-13T07:33:30,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:30,406 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31534528:(num 1731483210322) 2024-11-13T07:33:30,406 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:30,408 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:30,420 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409, exclude list is [], retry=0 2024-11-13T07:33:30,424 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:30,424 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:30,424 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:30,449 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 2024-11-13T07:33:30,450 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:30,697 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409, size=0 (0bytes) 2024-11-13T07:33:30,697 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 might be still open, length is 0 2024-11-13T07:33:30,697 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 2024-11-13T07:33:30,698 WARN [IPC Server handler 2 on default port 35647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T07:33:30,698 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 after 1ms 2024-11-13T07:33:33,586 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:45716 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45716 dst: /127.0.0.1:38619 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38619 remote=/127.0.0.1:45716]. Total timeout mills is 60000, 57080 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:33,587 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:40708 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40708 dst: /127.0.0.1:35659 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:33,587 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:33224 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33224 dst: /127.0.0.1:41959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:33,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741855_1032 (size=497775) 2024-11-13T07:33:33,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741855_1032 (size=497775) 2024-11-13T07:33:33,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741855_1032 (size=497775) 2024-11-13T07:33:34,082 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:33:34,132 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T07:33:34,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 after 4002ms 2024-11-13T07:33:34,727 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:34,729 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 took 4031ms 2024-11-13T07:33:34,733 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731483210409.temp 2024-11-13T07:33:34,760 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000000001-wal.1731483210409.temp 2024-11-13T07:33:34,844 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409; continuing. 2024-11-13T07:33:34,844 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 so closing down 2024-11-13T07:33:34,844 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:34,844 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:34,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741856_1033 (size=497783) 2024-11-13T07:33:34,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741856_1033 (size=497783) 2024-11-13T07:33:34,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741856_1033 (size=497783) 2024-11-13T07:33:34,848 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000000001-wal.1731483210409.temp (wrote 3002 edits, skipped 0 edits in 43 ms) 2024-11-13T07:33:34,850 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000000001-wal.1731483210409.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000003002 2024-11-13T07:33:34,851 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 122 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T07:33:34,851 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409, journal: Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409, size=0 (0bytes) at 1731483210697Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000000001-wal.1731483210409.temp at 1731483214760 (+4063 ms)Split 1024 edits, skipped 0 edits. at 1731483214798 (+38 ms)Split 2048 edits, skipped 0 edits. at 1731483214824 (+26 ms)Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 so closing down at 1731483214844 (+20 ms)3 split writer threads finished at 1731483214844Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000000001-wal.1731483210409.temp (wrote 3002 edits, skipped 0 edits in 43 ms) at 1731483214848 (+4 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000000001-wal.1731483210409.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000003002 at 1731483214850 (+2 ms)Processed 3002 edits across 1 Regions in 122 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409, size=0, length=0, corrupted=false, cancelled=false at 1731483214851 (+1 ms) 2024-11-13T07:33:34,853 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483210409 2024-11-13T07:33:34,855 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000003002 2024-11-13T07:33:34,855 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:34,857 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:34,874 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483214858, exclude list is [], retry=0 2024-11-13T07:33:34,877 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:34,877 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:34,878 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:34,903 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483214858 2024-11-13T07:33:34,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:34,903 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:34,905 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:34,906 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a53355305906a9a234d60b26d36ae3 columnFamilyName a 2024-11-13T07:33:34,906 DEBUG [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:34,907 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(327): Store=85a53355305906a9a234d60b26d36ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:34,907 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:34,908 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a53355305906a9a234d60b26d36ae3 columnFamilyName b 2024-11-13T07:33:34,908 DEBUG [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:34,909 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(327): Store=85a53355305906a9a234d60b26d36ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:34,909 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:34,910 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85a53355305906a9a234d60b26d36ae3 columnFamilyName c 2024-11-13T07:33:34,910 DEBUG [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:34,911 INFO [StoreOpener-85a53355305906a9a234d60b26d36ae3-1 {}] regionserver.HStore(327): Store=85a53355305906a9a234d60b26d36ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:34,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:34,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:34,914 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:34,915 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000003002 2024-11-13T07:33:34,943 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:34,987 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T07:33:35,300 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 85a53355305906a9a234d60b26d36ae3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T07:33:35,337 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/a/45086d4c9be846149f2139fea445448d is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731483210461/Put/seqid=0 2024-11-13T07:33:35,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741858_1035 (size=50463) 2024-11-13T07:33:35,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741858_1035 (size=50463) 2024-11-13T07:33:35,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741858_1035 (size=50463) 2024-11-13T07:33:35,345 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/a/45086d4c9be846149f2139fea445448d 2024-11-13T07:33:35,353 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/a/45086d4c9be846149f2139fea445448d as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/a/45086d4c9be846149f2139fea445448d 2024-11-13T07:33:35,360 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/a/45086d4c9be846149f2139fea445448d, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T07:33:35,361 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 85a53355305906a9a234d60b26d36ae3 in 60ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T07:33:35,380 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T07:33:35,380 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 85a53355305906a9a234d60b26d36ae3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T07:33:35,390 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/a/bb46de1773b443f294a65c565a860098 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731483210516/Put/seqid=0 2024-11-13T07:33:35,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741859_1036 (size=20072) 2024-11-13T07:33:35,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741859_1036 (size=20072) 2024-11-13T07:33:35,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741859_1036 (size=20072) 2024-11-13T07:33:35,402 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/a/bb46de1773b443f294a65c565a860098 2024-11-13T07:33:35,429 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/b/0676e04e249c411483db1a1b750b2b25 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731483210536/Put/seqid=0 2024-11-13T07:33:35,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741860_1037 (size=35835) 2024-11-13T07:33:35,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741860_1037 (size=35835) 2024-11-13T07:33:35,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741860_1037 (size=35835) 2024-11-13T07:33:35,437 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/b/0676e04e249c411483db1a1b750b2b25 2024-11-13T07:33:35,445 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/a/bb46de1773b443f294a65c565a860098 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/a/bb46de1773b443f294a65c565a860098 2024-11-13T07:33:35,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/a/bb46de1773b443f294a65c565a860098, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T07:33:35,455 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/b/0676e04e249c411483db1a1b750b2b25 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/b/0676e04e249c411483db1a1b750b2b25 2024-11-13T07:33:35,461 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/b/0676e04e249c411483db1a1b750b2b25, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T07:33:35,461 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 85a53355305906a9a234d60b26d36ae3 in 81ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T07:33:35,472 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T07:33:35,473 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 85a53355305906a9a234d60b26d36ae3 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T07:33:35,479 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/b/214b015fcb024847a0b3afb201f8ddf3 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731483210563/Put/seqid=0 2024-11-13T07:33:35,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741861_1038 (size=35082) 2024-11-13T07:33:35,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741861_1038 (size=35082) 2024-11-13T07:33:35,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741861_1038 (size=35082) 2024-11-13T07:33:35,488 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/b/214b015fcb024847a0b3afb201f8ddf3 2024-11-13T07:33:35,510 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/c/17b1fe761a1540b781caaa3778339701 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731483210603/Put/seqid=0 2024-11-13T07:33:35,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741862_1039 (size=20825) 2024-11-13T07:33:35,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741862_1039 (size=20825) 2024-11-13T07:33:35,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741862_1039 (size=20825) 2024-11-13T07:33:35,518 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/c/17b1fe761a1540b781caaa3778339701 2024-11-13T07:33:35,525 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/b/214b015fcb024847a0b3afb201f8ddf3 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/b/214b015fcb024847a0b3afb201f8ddf3 2024-11-13T07:33:35,532 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/b/214b015fcb024847a0b3afb201f8ddf3, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T07:33:35,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/c/17b1fe761a1540b781caaa3778339701 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/c/17b1fe761a1540b781caaa3778339701 2024-11-13T07:33:35,540 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/c/17b1fe761a1540b781caaa3778339701, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T07:33:35,540 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 85a53355305906a9a234d60b26d36ae3 in 67ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T07:33:35,550 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731483210664/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:35,553 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000003002 2024-11-13T07:33:35,554 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T07:33:35,554 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 85a53355305906a9a234d60b26d36ae3 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T07:33:35,561 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/c/ec9974c7b6a24313991239680704b2ac is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731483210613/Put/seqid=0 2024-11-13T07:33:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741863_1040 (size=50301) 2024-11-13T07:33:35,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741863_1040 (size=50301) 2024-11-13T07:33:35,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741863_1040 (size=50301) 2024-11-13T07:33:35,570 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/c/ec9974c7b6a24313991239680704b2ac 2024-11-13T07:33:35,576 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ec9974c7b6a24313991239680704b2ac 2024-11-13T07:33:35,578 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/.tmp/c/ec9974c7b6a24313991239680704b2ac as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/c/ec9974c7b6a24313991239680704b2ac 2024-11-13T07:33:35,584 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ec9974c7b6a24313991239680704b2ac 2024-11-13T07:33:35,584 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/c/ec9974c7b6a24313991239680704b2ac, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T07:33:35,585 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 85a53355305906a9a234d60b26d36ae3 in 31ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T07:33:35,586 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/0000000000000003002 2024-11-13T07:33:35,587 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:35,587 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:35,587 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T07:33:35,589 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 85a53355305906a9a234d60b26d36ae3 2024-11-13T07:33:35,591 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenIntoWAL/85a53355305906a9a234d60b26d36ae3/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T07:33:35,592 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 85a53355305906a9a234d60b26d36ae3; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71215705, jitterRate=0.0611966997385025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T07:33:35,593 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 85a53355305906a9a234d60b26d36ae3: Writing region info on filesystem at 1731483214903Initializing all the Stores at 1731483214905 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483214905Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483214905Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483214905Cleaning up temporary data from old regions at 1731483215587 (+682 ms)Region opened successfully at 1731483215593 (+6 ms) 2024-11-13T07:33:35,668 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 85a53355305906a9a234d60b26d36ae3, disabling compactions & flushes 2024-11-13T07:33:35,668 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:35,668 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:35,668 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. after waiting 0 ms 2024-11-13T07:33:35,668 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:35,670 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731483210316.85a53355305906a9a234d60b26d36ae3. 2024-11-13T07:33:35,670 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 85a53355305906a9a234d60b26d36ae3: Waiting for close lock at 1731483215668Disabling compacts and flushes for region at 1731483215668Disabling writes for close at 1731483215668Writing region close event to WAL at 1731483215670 (+2 ms)Closed at 1731483215670 2024-11-13T07:33:35,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741857_1034 (size=175) 2024-11-13T07:33:35,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741857_1034 (size=175) 2024-11-13T07:33:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741857_1034 (size=175) 2024-11-13T07:33:35,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:35,678 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731483214858) 2024-11-13T07:33:35,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=396 (was 384) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1090796390_22 at /127.0.0.1:41340 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1090796390_22 at /127.0.0.1:41364 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:40399 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40399 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:36663 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1090796390_22 at /127.0.0.1:56550 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35647 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1090796390_22 at /127.0.0.1:33568 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36663 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 337) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8697 (was 8760) 2024-11-13T07:33:35,702 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=396, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=8697 2024-11-13T07:33:35,717 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:35,719 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:35,720 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:35,723 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26066010, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-26066010, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:35,735 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26066010/hregion-26066010.1731483215723, exclude list is [], retry=0 2024-11-13T07:33:35,739 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:35,739 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:35,739 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:35,776 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26066010/hregion-26066010.1731483215723 2024-11-13T07:33:35,777 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:35,777 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b7d5b5121e11c15abd5e7bf7bade6a31, NAME => 'test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:35,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741865_1042 (size=43) 2024-11-13T07:33:35,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741865_1042 (size=43) 2024-11-13T07:33:35,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741865_1042 (size=43) 2024-11-13T07:33:35,789 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:35,790 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,792 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5b5121e11c15abd5e7bf7bade6a31 columnFamilyName a 2024-11-13T07:33:35,792 DEBUG [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:35,793 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(327): Store=b7d5b5121e11c15abd5e7bf7bade6a31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:35,793 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,794 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5b5121e11c15abd5e7bf7bade6a31 columnFamilyName b 2024-11-13T07:33:35,794 DEBUG [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:35,795 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(327): Store=b7d5b5121e11c15abd5e7bf7bade6a31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:35,795 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,796 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5b5121e11c15abd5e7bf7bade6a31 columnFamilyName c 2024-11-13T07:33:35,796 DEBUG [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:35,797 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(327): Store=b7d5b5121e11c15abd5e7bf7bade6a31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:35,797 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,800 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,800 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,800 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:35,802 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:35,804 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:35,805 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b7d5b5121e11c15abd5e7bf7bade6a31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61763499, jitterRate=-0.07965214550495148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:35,805 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b7d5b5121e11c15abd5e7bf7bade6a31: Writing region info on filesystem at 1731483215789Initializing all the Stores at 1731483215790 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483215790Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483215790Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483215790Cleaning up temporary data from old regions at 1731483215800 (+10 ms)Region opened successfully at 1731483215805 (+5 ms) 2024-11-13T07:33:35,805 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b7d5b5121e11c15abd5e7bf7bade6a31, disabling compactions & flushes 2024-11-13T07:33:35,806 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:35,806 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:35,806 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. after waiting 0 ms 2024-11-13T07:33:35,806 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:35,806 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:35,806 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b7d5b5121e11c15abd5e7bf7bade6a31: Waiting for close lock at 1731483215805Disabling compacts and flushes for region at 1731483215805Disabling writes for close at 1731483215806 (+1 ms)Writing region close event to WAL at 1731483215806Closed at 1731483215806 2024-11-13T07:33:35,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741864_1041 (size=175) 2024-11-13T07:33:35,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741864_1041 (size=175) 2024-11-13T07:33:35,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741864_1041 (size=175) 2024-11-13T07:33:35,814 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:35,814 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26066010:(num 1731483215723) 2024-11-13T07:33:35,815 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:35,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:35,836 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818, exclude list is [], retry=0 2024-11-13T07:33:35,839 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:35,839 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:35,840 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:35,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T07:33:35,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:35,860 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T07:33:35,860 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T07:33:35,860 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T07:33:35,860 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:35,861 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T07:33:35,861 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:35,877 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 2024-11-13T07:33:35,877 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:36,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741866_1043 (size=371388) 2024-11-13T07:33:36,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741866_1043 (size=371388) 2024-11-13T07:33:36,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741866_1043 (size=371388) 2024-11-13T07:33:36,097 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818, size=362.7 K (371388bytes) 2024-11-13T07:33:36,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 2024-11-13T07:33:36,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 after 0ms 2024-11-13T07:33:36,134 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:36,135 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 took 39ms 2024-11-13T07:33:36,139 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731483215818.temp 2024-11-13T07:33:36,161 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 so closing down 2024-11-13T07:33:36,161 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:36,178 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000000001-wal.1731483215818.temp 2024-11-13T07:33:36,196 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:36,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741867_1044 (size=371388) 2024-11-13T07:33:36,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741867_1044 (size=371388) 2024-11-13T07:33:36,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741867_1044 (size=371388) 2024-11-13T07:33:36,200 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000000001-wal.1731483215818.temp (wrote 3000 edits, skipped 0 edits in 18 ms) 2024-11-13T07:33:36,202 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000000001-wal.1731483215818.temp to hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003000 2024-11-13T07:33:36,203 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-13T07:33:36,203 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818, journal: Splitting hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818, size=362.7 K (371388bytes) at 1731483216097Split 1024 edits, skipped 0 edits. at 1731483216146 (+49 ms)Split 2048 edits, skipped 0 edits. at 1731483216154 (+8 ms)Finishing writing output for hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 so closing down at 1731483216161 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000000001-wal.1731483215818.temp at 1731483216178 (+17 ms)3 split writer threads finished at 1731483216196 (+18 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000000001-wal.1731483215818.temp (wrote 3000 edits, skipped 0 edits in 18 ms) at 1731483216201 (+5 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000000001-wal.1731483215818.temp to hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003000 at 1731483216202 (+1 ms)Processed 3000 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731483216203 (+1 ms) 2024-11-13T07:33:36,205 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483215818 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483215818 2024-11-13T07:33:36,206 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003000 2024-11-13T07:33:36,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:36,208 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:36,221 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208, exclude list is [], retry=0 2024-11-13T07:33:36,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:36,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:36,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:36,252 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 2024-11-13T07:33:36,252 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:33:36,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741868_1045 (size=371515) 2024-11-13T07:33:36,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741868_1045 (size=371515) 2024-11-13T07:33:36,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741868_1045 (size=371515) 2024-11-13T07:33:36,414 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208, size=362.8 K (371515bytes) 2024-11-13T07:33:36,414 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 2024-11-13T07:33:36,415 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 after 0ms 2024-11-13T07:33:36,454 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:36,456 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 took 41ms 2024-11-13T07:33:36,459 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731483216208.temp 2024-11-13T07:33:36,494 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 so closing down 2024-11-13T07:33:36,494 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:36,500 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003001-wal.1731483216208.temp 2024-11-13T07:33:36,516 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:36,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741869_1046 (size=371515) 2024-11-13T07:33:36,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741869_1046 (size=371515) 2024-11-13T07:33:36,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741869_1046 (size=371515) 2024-11-13T07:33:36,525 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003001-wal.1731483216208.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-13T07:33:36,527 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003001-wal.1731483216208.temp to hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000006000 2024-11-13T07:33:36,527 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-13T07:33:36,527 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208, journal: Splitting hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208, size=362.8 K (371515bytes) at 1731483216414Split 1024 edits, skipped 0 edits. at 1731483216473 (+59 ms)Split 2048 edits, skipped 0 edits. at 1731483216485 (+12 ms)Finishing writing output for hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 so closing down at 1731483216494 (+9 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003001-wal.1731483216208.temp at 1731483216500 (+6 ms)3 split writer threads finished at 1731483216516 (+16 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003001-wal.1731483216208.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1731483216525 (+9 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003001-wal.1731483216208.temp to hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000006000 at 1731483216527 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731483216527 2024-11-13T07:33:36,529 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216208 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483216208 2024-11-13T07:33:36,531 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000006000 2024-11-13T07:33:36,531 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:36,533 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/test2727-manual,16010,1731483215716, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:36,554 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216534, exclude list is [], retry=0 2024-11-13T07:33:36,558 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:36,558 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:36,559 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:36,592 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731483215716/wal.1731483216534 2024-11-13T07:33:36,592 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:33:36,593 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b7d5b5121e11c15abd5e7bf7bade6a31, NAME => 'test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:36,593 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:36,593 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,593 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,595 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,596 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5b5121e11c15abd5e7bf7bade6a31 columnFamilyName a 2024-11-13T07:33:36,596 DEBUG [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:36,597 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(327): Store=b7d5b5121e11c15abd5e7bf7bade6a31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:36,597 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,599 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5b5121e11c15abd5e7bf7bade6a31 columnFamilyName b 2024-11-13T07:33:36,599 DEBUG [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:36,599 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(327): Store=b7d5b5121e11c15abd5e7bf7bade6a31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:36,600 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,601 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5b5121e11c15abd5e7bf7bade6a31 columnFamilyName c 2024-11-13T07:33:36,601 DEBUG [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:36,602 INFO [StoreOpener-b7d5b5121e11c15abd5e7bf7bade6a31-1 {}] regionserver.HStore(327): Store=b7d5b5121e11c15abd5e7bf7bade6a31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:36,603 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,603 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,607 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003000 2024-11-13T07:33:36,638 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:36,674 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003000 2024-11-13T07:33:36,676 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000006000 2024-11-13T07:33:36,702 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:36,736 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000006000 2024-11-13T07:33:36,736 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b7d5b5121e11c15abd5e7bf7bade6a31 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T07:33:36,787 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/a/e93edc49069742c7a205c49f1d052e96 is 41, key is test2727/a:100/1731483216257/Put/seqid=0 2024-11-13T07:33:36,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741871_1048 (size=84227) 2024-11-13T07:33:36,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741871_1048 (size=84227) 2024-11-13T07:33:36,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741871_1048 (size=84227) 2024-11-13T07:33:36,794 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/a/e93edc49069742c7a205c49f1d052e96 2024-11-13T07:33:36,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/b/f04f4d9ad6ae4617abdf4c4d9f8cdb02 is 41, key is test2727/b:100/1731483216303/Put/seqid=0 2024-11-13T07:33:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741872_1049 (size=84609) 2024-11-13T07:33:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741872_1049 (size=84609) 2024-11-13T07:33:36,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741872_1049 (size=84609) 2024-11-13T07:33:36,835 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/b/f04f4d9ad6ae4617abdf4c4d9f8cdb02 2024-11-13T07:33:36,857 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/c/58d671ee9dcc48ce9f6d019ac177f603 is 41, key is test2727/c:100/1731483216352/Put/seqid=0 2024-11-13T07:33:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741873_1050 (size=84609) 2024-11-13T07:33:36,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741873_1050 (size=84609) 2024-11-13T07:33:36,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741873_1050 (size=84609) 2024-11-13T07:33:36,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/c/58d671ee9dcc48ce9f6d019ac177f603 2024-11-13T07:33:36,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/a/e93edc49069742c7a205c49f1d052e96 as hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/a/e93edc49069742c7a205c49f1d052e96 2024-11-13T07:33:36,880 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/a/e93edc49069742c7a205c49f1d052e96, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T07:33:36,882 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/b/f04f4d9ad6ae4617abdf4c4d9f8cdb02 as hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/b/f04f4d9ad6ae4617abdf4c4d9f8cdb02 2024-11-13T07:33:36,890 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/b/f04f4d9ad6ae4617abdf4c4d9f8cdb02, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T07:33:36,892 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/.tmp/c/58d671ee9dcc48ce9f6d019ac177f603 as hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/c/58d671ee9dcc48ce9f6d019ac177f603 2024-11-13T07:33:36,899 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/c/58d671ee9dcc48ce9f6d019ac177f603, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T07:33:36,900 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for b7d5b5121e11c15abd5e7bf7bade6a31 in 163ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T07:33:36,901 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000003000 2024-11-13T07:33:36,901 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/0000000000000006000 2024-11-13T07:33:36,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,904 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:36,906 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b7d5b5121e11c15abd5e7bf7bade6a31 2024-11-13T07:33:36,909 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/test2727/b7d5b5121e11c15abd5e7bf7bade6a31/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T07:33:36,910 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b7d5b5121e11c15abd5e7bf7bade6a31; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66809103, jitterRate=-0.004466786980628967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:36,911 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b7d5b5121e11c15abd5e7bf7bade6a31: Writing region info on filesystem at 1731483216593Initializing all the Stores at 1731483216594 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483216594Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483216594Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483216595 (+1 ms)Obtaining lock to block concurrent updates at 1731483216736 (+141 ms)Preparing flush snapshotting stores in b7d5b5121e11c15abd5e7bf7bade6a31 at 1731483216736Finished memstore snapshotting test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731483216736Flushing stores of test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. at 1731483216736Flushing b7d5b5121e11c15abd5e7bf7bade6a31/a: creating writer at 1731483216736Flushing b7d5b5121e11c15abd5e7bf7bade6a31/a: appending metadata at 1731483216786 (+50 ms)Flushing b7d5b5121e11c15abd5e7bf7bade6a31/a: closing flushed file at 1731483216786Flushing b7d5b5121e11c15abd5e7bf7bade6a31/b: creating writer at 1731483216801 (+15 ms)Flushing b7d5b5121e11c15abd5e7bf7bade6a31/b: appending metadata at 1731483216826 (+25 ms)Flushing b7d5b5121e11c15abd5e7bf7bade6a31/b: closing flushed file at 1731483216826Flushing b7d5b5121e11c15abd5e7bf7bade6a31/c: creating writer at 1731483216841 (+15 ms)Flushing b7d5b5121e11c15abd5e7bf7bade6a31/c: appending metadata at 1731483216856 (+15 ms)Flushing b7d5b5121e11c15abd5e7bf7bade6a31/c: closing flushed file at 1731483216856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c234ee6: reopening flushed file at 1731483216871 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@225c61b1: reopening flushed file at 1731483216880 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@622162b1: reopening flushed file at 1731483216890 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for b7d5b5121e11c15abd5e7bf7bade6a31 in 163ms, sequenceid=6000, compaction requested=false; wal=null at 1731483216900 (+10 ms)Cleaning up temporary data from old regions at 1731483216903 (+3 ms)Region opened successfully at 1731483216911 (+8 ms) 2024-11-13T07:33:36,913 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T07:33:36,913 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b7d5b5121e11c15abd5e7bf7bade6a31, disabling compactions & flushes 2024-11-13T07:33:36,913 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:36,913 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:36,913 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. after waiting 0 ms 2024-11-13T07:33:36,913 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:36,914 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731483215718.b7d5b5121e11c15abd5e7bf7bade6a31. 2024-11-13T07:33:36,914 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b7d5b5121e11c15abd5e7bf7bade6a31: Waiting for close lock at 1731483216913Disabling compacts and flushes for region at 1731483216913Disabling writes for close at 1731483216913Writing region close event to WAL at 1731483216914 (+1 ms)Closed at 1731483216914 2024-11-13T07:33:36,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741870_1047 (size=175) 2024-11-13T07:33:36,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741870_1047 (size=175) 2024-11-13T07:33:36,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741870_1047 (size=175) 2024-11-13T07:33:36,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:36,921 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731483216534) 2024-11-13T07:33:36,934 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402 (was 396) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:33662 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:41454 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:56664 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=8467 (was 8697) 2024-11-13T07:33:36,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=8466 2024-11-13T07:33:36,959 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:36,965 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:36,966 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731483216966 2024-11-13T07:33:36,997 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966 2024-11-13T07:33:37,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:37,002 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 25ed677f87e855e89bde863c08677e88, NAME => 'testSequentialEditLogSeqNum,,1731483216960.25ed677f87e855e89bde863c08677e88.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:37,002 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731483216960.25ed677f87e855e89bde863c08677e88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:37,002 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,003 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,003 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88 doesn't exist for region: 25ed677f87e855e89bde863c08677e88 on table testSequentialEditLogSeqNum 2024-11-13T07:33:37,004 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 25ed677f87e855e89bde863c08677e88 on table testSequentialEditLogSeqNum 2024-11-13T07:33:37,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741875_1052 (size=62) 2024-11-13T07:33:37,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741875_1052 (size=62) 2024-11-13T07:33:37,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741875_1052 (size=62) 2024-11-13T07:33:37,019 INFO [StoreOpener-25ed677f87e855e89bde863c08677e88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,021 INFO [StoreOpener-25ed677f87e855e89bde863c08677e88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25ed677f87e855e89bde863c08677e88 columnFamilyName a 2024-11-13T07:33:37,021 DEBUG [StoreOpener-25ed677f87e855e89bde863c08677e88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,022 INFO [StoreOpener-25ed677f87e855e89bde863c08677e88-1 {}] regionserver.HStore(327): Store=25ed677f87e855e89bde863c08677e88/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,022 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,023 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,023 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,025 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 25ed677f87e855e89bde863c08677e88 2024-11-13T07:33:37,027 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:37,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 25ed677f87e855e89bde863c08677e88; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59628345, jitterRate=-0.11146841943264008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:37,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 25ed677f87e855e89bde863c08677e88: Writing region info on filesystem at 1731483217003Initializing all the Stores at 1731483217019 (+16 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217019Cleaning up temporary data from old regions at 1731483217023 (+4 ms)Region opened successfully at 1731483217028 (+5 ms) 2024-11-13T07:33:37,039 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 25ed677f87e855e89bde863c08677e88 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T07:33:37,059 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/.tmp/a/4fd4546375bc426b86bbe9f0c1443a99 is 81, key is testSequentialEditLogSeqNum/a:x0/1731483217028/Put/seqid=0 2024-11-13T07:33:37,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741876_1053 (size=5833) 2024-11-13T07:33:37,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741876_1053 (size=5833) 2024-11-13T07:33:37,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741876_1053 (size=5833) 2024-11-13T07:33:37,072 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/.tmp/a/4fd4546375bc426b86bbe9f0c1443a99 2024-11-13T07:33:37,079 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/.tmp/a/4fd4546375bc426b86bbe9f0c1443a99 as hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/a/4fd4546375bc426b86bbe9f0c1443a99 2024-11-13T07:33:37,086 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/a/4fd4546375bc426b86bbe9f0c1443a99, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T07:33:37,087 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 25ed677f87e855e89bde863c08677e88 in 48ms, sequenceid=13, compaction requested=false 2024-11-13T07:33:37,087 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 25ed677f87e855e89bde863c08677e88: 2024-11-13T07:33:37,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T07:33:37,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T07:33:37,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T07:33:37,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T07:33:37,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T07:33:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741874_1051 (size=3256) 2024-11-13T07:33:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741874_1051 (size=3256) 2024-11-13T07:33:37,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741874_1051 (size=3256) 2024-11-13T07:33:37,111 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966, size=3.2 K (3256bytes) 2024-11-13T07:33:37,112 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966 2024-11-13T07:33:37,112 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966 after 0ms 2024-11-13T07:33:37,140 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:37,140 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966 took 29ms 2024-11-13T07:33:37,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966 so closing down 2024-11-13T07:33:37,144 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:37,144 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731483216966.temp 2024-11-13T07:33:37,168 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000003-wal.1731483216966.temp 2024-11-13T07:33:37,169 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:37,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741877_1054 (size=2650) 2024-11-13T07:33:37,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741877_1054 (size=2650) 2024-11-13T07:33:37,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741877_1054 (size=2650) 2024-11-13T07:33:37,176 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000003-wal.1731483216966.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:37,177 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000003-wal.1731483216966.temp to hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000020 2024-11-13T07:33:37,178 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-13T07:33:37,178 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966, journal: Splitting hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966, size=3.2 K (3256bytes) at 1731483217112Finishing writing output for hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966 so closing down at 1731483217143 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000003-wal.1731483216966.temp at 1731483217168 (+25 ms)3 split writer threads finished at 1731483217169 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000003-wal.1731483216966.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731483217176 (+7 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000003-wal.1731483216966.temp to hdfs://localhost:35647/hbase/data/default/testSequentialEditLogSeqNum/25ed677f87e855e89bde863c08677e88/recovered.edits/0000000000000000020 at 1731483217177 (+1 ms)Processed 17 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35647/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731483216958/wal.1731483216966, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731483217178 (+1 ms) 2024-11-13T07:33:37,192 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:41454 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:56664 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=8460 (was 8466) 2024-11-13T07:33:37,204 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=8458 2024-11-13T07:33:37,219 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:37,221 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:37,253 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:37,256 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57731998, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-57731998, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:37,268 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57731998/hregion-57731998.1731483217256, exclude list is [], retry=0 2024-11-13T07:33:37,272 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:37,272 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:37,272 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:37,299 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57731998/hregion-57731998.1731483217256 2024-11-13T07:33:37,299 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:37,299 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 075da07df828a2ce719bf2136c0c77d6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:37,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741879_1056 (size=70) 2024-11-13T07:33:37,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741879_1056 (size=70) 2024-11-13T07:33:37,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741879_1056 (size=70) 2024-11-13T07:33:37,313 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:37,314 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,316 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName a 2024-11-13T07:33:37,316 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,317 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,317 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,318 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName b 2024-11-13T07:33:37,318 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,319 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,319 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,320 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName c 2024-11-13T07:33:37,320 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,321 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,321 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,322 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,323 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,324 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,324 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,324 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:37,326 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,328 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:37,329 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 075da07df828a2ce719bf2136c0c77d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64164005, jitterRate=-0.043881818652153015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 075da07df828a2ce719bf2136c0c77d6: Writing region info on filesystem at 1731483217313Initializing all the Stores at 1731483217314 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217314Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217314Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217314Cleaning up temporary data from old regions at 1731483217324 (+10 ms)Region opened successfully at 1731483217329 (+5 ms) 2024-11-13T07:33:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 075da07df828a2ce719bf2136c0c77d6, disabling compactions & flushes 2024-11-13T07:33:37,330 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. after waiting 0 ms 2024-11-13T07:33:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:37,330 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:37,331 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 075da07df828a2ce719bf2136c0c77d6: Waiting for close lock at 1731483217330Disabling compacts and flushes for region at 1731483217330Disabling writes for close at 1731483217330Writing region close event to WAL at 1731483217330Closed at 1731483217330 2024-11-13T07:33:37,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741878_1055 (size=175) 2024-11-13T07:33:37,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741878_1055 (size=175) 2024-11-13T07:33:37,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741878_1055 (size=175) 2024-11-13T07:33:37,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:37,336 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57731998:(num 1731483217256) 2024-11-13T07:33:37,336 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:37,338 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:37,350 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338, exclude list is [], retry=0 2024-11-13T07:33:37,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:37,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:37,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:37,379 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 2024-11-13T07:33:37,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:37,380 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 075da07df828a2ce719bf2136c0c77d6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:37,380 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:37,380 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,380 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,382 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,383 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName a 2024-11-13T07:33:37,383 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,383 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,383 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,384 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName b 2024-11-13T07:33:37,384 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,385 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,385 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,386 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName c 2024-11-13T07:33:37,386 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:37,386 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:37,386 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,388 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,389 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,389 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,390 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:37,391 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:37,392 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 075da07df828a2ce719bf2136c0c77d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60836519, jitterRate=-0.09346522390842438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:37,393 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 075da07df828a2ce719bf2136c0c77d6: Writing region info on filesystem at 1731483217380Initializing all the Stores at 1731483217381 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217381Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217382 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483217382Cleaning up temporary data from old regions at 1731483217389 (+7 ms)Region opened successfully at 1731483217393 (+4 ms) 2024-11-13T07:33:37,397 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731483217396/Put/seqid=0 2024-11-13T07:33:37,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741881_1058 (size=4826) 2024-11-13T07:33:37,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741881_1058 (size=4826) 2024-11-13T07:33:37,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741881_1058 (size=4826) 2024-11-13T07:33:37,404 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 075da07df828a2ce719bf2136c0c77d6/a 2024-11-13T07:33:37,412 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T07:33:37,412 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T07:33:37,413 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 075da07df828a2ce719bf2136c0c77d6: 2024-11-13T07:33:37,415 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/370f6e1e697b4223a17bb32178805d80_SeqId_3_ 2024-11-13T07:33:37,416 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 075da07df828a2ce719bf2136c0c77d6/a as hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/370f6e1e697b4223a17bb32178805d80_SeqId_3_ - updating store file list. 2024-11-13T07:33:37,422 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 370f6e1e697b4223a17bb32178805d80_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T07:33:37,422 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/370f6e1e697b4223a17bb32178805d80_SeqId_3_ into 075da07df828a2ce719bf2136c0c77d6/a 2024-11-13T07:33:37,423 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35647/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 075da07df828a2ce719bf2136c0c77d6/a (new location: hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/370f6e1e697b4223a17bb32178805d80_SeqId_3_) 2024-11-13T07:33:37,460 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338, size=0 (0bytes) 2024-11-13T07:33:37,460 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 might be still open, length is 0 2024-11-13T07:33:37,460 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 2024-11-13T07:33:37,461 WARN [IPC Server handler 3 on default port 35647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T07:33:37,461 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 after 1ms 2024-11-13T07:33:39,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:33782 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33782 dst: /127.0.0.1:35659 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35659 remote=/127.0.0.1:33782]. Total timeout mills is 60000, 57861 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:39,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:41568 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41568 dst: /127.0.0.1:41959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:39,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:56728 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56728 dst: /127.0.0.1:38619 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741880_1059 (size=606) 2024-11-13T07:33:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741880_1059 (size=606) 2024-11-13T07:33:41,462 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 after 4002ms 2024-11-13T07:33:41,494 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:41,494 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 took 4034ms 2024-11-13T07:33:41,496 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338; continuing. 2024-11-13T07:33:41,496 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 so closing down 2024-11-13T07:33:41,496 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:41,498 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731483217338.temp 2024-11-13T07:33:41,526 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005-wal.1731483217338.temp 2024-11-13T07:33:41,526 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:41,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741882_1060 (size=329) 2024-11-13T07:33:41,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741882_1060 (size=329) 2024-11-13T07:33:41,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741882_1060 (size=329) 2024-11-13T07:33:41,534 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005-wal.1731483217338.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:41,535 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005-wal.1731483217338.temp to hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005 2024-11-13T07:33:41,536 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 42 ms; skipped=1; WAL=hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T07:33:41,536 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338, journal: Splitting hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338, size=0 (0bytes) at 1731483217460Finishing writing output for hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 so closing down at 1731483221496 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005-wal.1731483217338.temp at 1731483221526 (+30 ms)3 split writer threads finished at 1731483221526Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005-wal.1731483217338.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731483221534 (+8 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005-wal.1731483217338.temp to hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005 at 1731483221535 (+1 ms)Processed 2 edits across 1 Regions in 42 ms; skipped=1; WAL=hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338, size=0, length=0, corrupted=false, cancelled=false at 1731483221536 (+1 ms) 2024-11-13T07:33:41,537 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483217338 2024-11-13T07:33:41,538 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005 2024-11-13T07:33:41,538 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:41,540 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:41,552 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483221540, exclude list is [], retry=0 2024-11-13T07:33:41,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:41,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:41,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:41,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483221540 2024-11-13T07:33:41,581 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:41,581 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 075da07df828a2ce719bf2136c0c77d6, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:41,581 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:41,581 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,581 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,583 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,584 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName a 2024-11-13T07:33:41,584 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:41,591 DEBUG [StoreFileOpener-075da07df828a2ce719bf2136c0c77d6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 370f6e1e697b4223a17bb32178805d80_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T07:33:41,591 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/370f6e1e697b4223a17bb32178805d80_SeqId_3_ 2024-11-13T07:33:41,591 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:41,591 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,592 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName b 2024-11-13T07:33:41,592 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:41,593 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:41,593 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,594 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075da07df828a2ce719bf2136c0c77d6 columnFamilyName c 2024-11-13T07:33:41,594 DEBUG [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:41,595 INFO [StoreOpener-075da07df828a2ce719bf2136c0c77d6-1 {}] regionserver.HStore(327): Store=075da07df828a2ce719bf2136c0c77d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:41,595 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,596 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,598 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005 2024-11-13T07:33:41,624 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:41,625 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005 2024-11-13T07:33:41,625 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 075da07df828a2ce719bf2136c0c77d6 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T07:33:41,639 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/.tmp/a/64cb05bdba154b74b0c1357625ad6b51 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731483217427/Put/seqid=0 2024-11-13T07:33:41,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741884_1062 (size=5149) 2024-11-13T07:33:41,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741884_1062 (size=5149) 2024-11-13T07:33:41,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741884_1062 (size=5149) 2024-11-13T07:33:41,646 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/.tmp/a/64cb05bdba154b74b0c1357625ad6b51 2024-11-13T07:33:41,651 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/.tmp/a/64cb05bdba154b74b0c1357625ad6b51 as hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/64cb05bdba154b74b0c1357625ad6b51 2024-11-13T07:33:41,657 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/a/64cb05bdba154b74b0c1357625ad6b51, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T07:33:41,657 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 075da07df828a2ce719bf2136c0c77d6 in 32ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T07:33:41,658 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/0000000000000000005 2024-11-13T07:33:41,659 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,659 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,660 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:41,662 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 075da07df828a2ce719bf2136c0c77d6 2024-11-13T07:33:41,664 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/075da07df828a2ce719bf2136c0c77d6/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:33:41,665 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 075da07df828a2ce719bf2136c0c77d6; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69442187, jitterRate=0.034769222140312195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:41,665 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 075da07df828a2ce719bf2136c0c77d6: Writing region info on filesystem at 1731483221581Initializing all the Stores at 1731483221582 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483221582Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483221583 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483221583Obtaining lock to block concurrent updates at 1731483221625 (+42 ms)Preparing flush snapshotting stores in 075da07df828a2ce719bf2136c0c77d6 at 1731483221625Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731483221625Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. at 1731483221625Flushing 075da07df828a2ce719bf2136c0c77d6/a: creating writer at 1731483221626 (+1 ms)Flushing 075da07df828a2ce719bf2136c0c77d6/a: appending metadata at 1731483221638 (+12 ms)Flushing 075da07df828a2ce719bf2136c0c77d6/a: closing flushed file at 1731483221639 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67718335: reopening flushed file at 1731483221650 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 075da07df828a2ce719bf2136c0c77d6 in 32ms, sequenceid=5, compaction requested=false; wal=null at 1731483221657 (+7 ms)Cleaning up temporary data from old regions at 1731483221659 (+2 ms)Region opened successfully at 1731483221665 (+6 ms) 2024-11-13T07:33:41,669 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 075da07df828a2ce719bf2136c0c77d6, disabling compactions & flushes 2024-11-13T07:33:41,669 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:41,669 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:41,669 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. after waiting 0 ms 2024-11-13T07:33:41,669 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:41,669 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731483217219.075da07df828a2ce719bf2136c0c77d6. 2024-11-13T07:33:41,670 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 075da07df828a2ce719bf2136c0c77d6: Waiting for close lock at 1731483221668Disabling compacts and flushes for region at 1731483221668Disabling writes for close at 1731483221669 (+1 ms)Writing region close event to WAL at 1731483221669Closed at 1731483221669 2024-11-13T07:33:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741883_1061 (size=175) 2024-11-13T07:33:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741883_1061 (size=175) 2024-11-13T07:33:41,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741883_1061 (size=175) 2024-11-13T07:33:41,674 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:41,674 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731483221540) 2024-11-13T07:33:41,685 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 407) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-918404690_22 at /127.0.0.1:33828 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-918404690_22 at /127.0.0.1:41596 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35647 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-918404690_22 at /127.0.0.1:56762 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=8460 (was 8458) - AvailableMemoryMB LEAK? - 2024-11-13T07:33:41,697 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=8458 2024-11-13T07:33:41,712 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:41,717 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T07:33:41,721 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2998b885bdef,44967,1731483205524 2024-11-13T07:33:41,723 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1982e98d 2024-11-13T07:33:41,724 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T07:33:41,726 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T07:33:41,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T07:33:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T07:33:41,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T07:33:41,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T07:33:41,742 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:41,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T07:33:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:33:41,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741885_1063 (size=694) 2024-11-13T07:33:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741885_1063 (size=694) 2024-11-13T07:33:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741885_1063 (size=694) 2024-11-13T07:33:41,757 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b 2024-11-13T07:33:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741886_1064 (size=77) 2024-11-13T07:33:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741886_1064 (size=77) 2024-11-13T07:33:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741886_1064 (size=77) 2024-11-13T07:33:41,766 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:41,766 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:33:41,766 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:41,766 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:41,766 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:33:41,766 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:41,766 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:41,766 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483221766Disabling compacts and flushes for region at 1731483221766Disabling writes for close at 1731483221766Writing region close event to WAL at 1731483221766Closed at 1731483221766 2024-11-13T07:33:41,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T07:33:41,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731483221768"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731483221768"}]},"ts":"1731483221768"} 2024-11-13T07:33:41,777 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T07:33:41,779 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T07:33:41,781 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731483221779"}]},"ts":"1731483221779"} 2024-11-13T07:33:41,784 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T07:33:41,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T07:33:41,786 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:33:41,786 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:33:41,786 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T07:33:41,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:33:41,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN}] 2024-11-13T07:33:41,790 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN 2024-11-13T07:33:41,791 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN; state=OFFLINE, location=2998b885bdef,38607,1731483206428; forceNewPlan=false, retain=false 2024-11-13T07:33:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:33:41,948 INFO [2998b885bdef:44967 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T07:33:41,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPENING, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:41,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN because future has completed 2024-11-13T07:33:41,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428}] 2024-11-13T07:33:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:33:42,117 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,117 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:42,117 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,118 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:42,118 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,118 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,120 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,121 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf1 2024-11-13T07:33:42,121 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:42,122 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:42,122 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,124 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf2 2024-11-13T07:33:42,124 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:42,124 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:42,125 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,126 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,126 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,127 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,127 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,128 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T07:33:42,130 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,133 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:42,134 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 47901928f170b169c8ff6f7dea663386; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59888052, jitterRate=-0.10759848356246948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T07:33:42,134 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,135 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 47901928f170b169c8ff6f7dea663386: Running coprocessor pre-open hook at 1731483222118Writing region info on filesystem at 1731483222118Initializing all the Stores at 1731483222119 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483222119Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483222119Cleaning up temporary data from old regions at 1731483222127 (+8 ms)Running coprocessor post-open hooks at 1731483222134 (+7 ms)Region opened successfully at 1731483222135 (+1 ms) 2024-11-13T07:33:42,137 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., pid=6, masterSystemTime=1731483222106 2024-11-13T07:33:42,140 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,140 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,140 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:42,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428 because future has completed 2024-11-13T07:33:42,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T07:33:42,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428 in 192 msec 2024-11-13T07:33:42,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T07:33:42,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN in 360 msec 2024-11-13T07:33:42,154 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T07:33:42,154 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731483222154"}]},"ts":"1731483222154"} 2024-11-13T07:33:42,157 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T07:33:42,158 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T07:33:42,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 425 msec 2024-11-13T07:33:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:33:42,374 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T07:33:42,374 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T07:33:42,377 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:33:42,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T07:33:42,384 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:33:42,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T07:33:42,397 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=2] 2024-11-13T07:33:42,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=47901928f170b169c8ff6f7dea663386, source=2998b885bdef,38607,1731483206428, destination=2998b885bdef,39583,1731483206286, warming up region on 2998b885bdef,39583,1731483206286 2024-11-13T07:33:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:33:42,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=47901928f170b169c8ff6f7dea663386, source=2998b885bdef,38607,1731483206428, destination=2998b885bdef,39583,1731483206286, running balancer 2024-11-13T07:33:42,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55095, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:33:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE 2024-11-13T07:33:42,416 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE 2024-11-13T07:33:42,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=CLOSING, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:42,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(7855): Warmup {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:42,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE because future has completed 2024-11-13T07:33:42,422 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,423 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T07:33:42,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428}] 2024-11-13T07:33:42,423 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf1 2024-11-13T07:33:42,423 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:42,424 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:42,424 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,425 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf2 2024-11-13T07:33:42,425 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:42,426 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:33:42,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:33:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483222426Disabling compacts and flushes for region at 1731483222426Disabling writes for close at 1731483222426Writing region close event to WAL at 1731483222427 (+1 ms)Closed at 1731483222427 2024-11-13T07:33:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T07:33:42,582 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,582 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T07:33:42,583 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:33:42,583 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,583 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,584 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:33:42,584 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,584 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 47901928f170b169c8ff6f7dea663386 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T07:33:42,606 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/d002e675e2cd4cb08f956a316f72a8e6 is 35, key is r1/cf1:q/1731483222400/Put/seqid=0 2024-11-13T07:33:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741887_1065 (size=4783) 2024-11-13T07:33:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741887_1065 (size=4783) 2024-11-13T07:33:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741887_1065 (size=4783) 2024-11-13T07:33:42,614 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/d002e675e2cd4cb08f956a316f72a8e6 2024-11-13T07:33:42,622 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/d002e675e2cd4cb08f956a316f72a8e6 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6 2024-11-13T07:33:42,629 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T07:33:42,630 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 47901928f170b169c8ff6f7dea663386 in 46ms, sequenceid=5, compaction requested=false 2024-11-13T07:33:42,630 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T07:33:42,636 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:33:42,639 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,639 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483222583Running coprocessor pre-close hooks at 1731483222583Disabling compacts and flushes for region at 1731483222583Disabling writes for close at 1731483222584 (+1 ms)Obtaining lock to block concurrent updates at 1731483222584Preparing flush snapshotting stores in 47901928f170b169c8ff6f7dea663386 at 1731483222584Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731483222584Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. at 1731483222585 (+1 ms)Flushing 47901928f170b169c8ff6f7dea663386/cf1: creating writer at 1731483222585Flushing 47901928f170b169c8ff6f7dea663386/cf1: appending metadata at 1731483222605 (+20 ms)Flushing 47901928f170b169c8ff6f7dea663386/cf1: closing flushed file at 1731483222605Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26356e12: reopening flushed file at 1731483222621 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 47901928f170b169c8ff6f7dea663386 in 46ms, sequenceid=5, compaction requested=false at 1731483222630 (+9 ms)Writing region close event to WAL at 1731483222632 (+2 ms)Running coprocessor post-close hooks at 1731483222637 (+5 ms)Closed at 1731483222639 (+2 ms) 2024-11-13T07:33:42,640 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 47901928f170b169c8ff6f7dea663386 move to 2998b885bdef,39583,1731483206286 record at close sequenceid=5 2024-11-13T07:33:42,643 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,644 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=CLOSED 2024-11-13T07:33:42,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428 because future has completed 2024-11-13T07:33:42,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T07:33:42,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428 in 225 msec 2024-11-13T07:33:42,651 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE; state=CLOSED, location=2998b885bdef,39583,1731483206286; forceNewPlan=false, retain=false 2024-11-13T07:33:42,802 INFO [2998b885bdef:44967 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T07:33:42,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPENING, regionLocation=2998b885bdef,39583,1731483206286 2024-11-13T07:33:42,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE because future has completed 2024-11-13T07:33:42,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,39583,1731483206286}] 2024-11-13T07:33:42,968 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,968 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:42,968 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,968 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:42,968 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,969 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,970 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,972 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf1 2024-11-13T07:33:42,972 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:42,980 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6 2024-11-13T07:33:42,980 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:42,980 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,982 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf2 2024-11-13T07:33:42,982 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:42,983 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:42,983 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,984 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,986 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,987 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,987 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,988 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T07:33:42,990 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,991 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 47901928f170b169c8ff6f7dea663386; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62923469, jitterRate=-0.06236724555492401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T07:33:42,991 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:42,992 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 47901928f170b169c8ff6f7dea663386: Running coprocessor pre-open hook at 1731483222969Writing region info on filesystem at 1731483222969Initializing all the Stores at 1731483222970 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483222970Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483222970Cleaning up temporary data from old regions at 1731483222987 (+17 ms)Running coprocessor post-open hooks at 1731483222991 (+4 ms)Region opened successfully at 1731483222992 (+1 ms) 2024-11-13T07:33:42,994 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., pid=9, masterSystemTime=1731483222963 2024-11-13T07:33:42,996 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,996 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:42,997 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPEN, openSeqNum=9, regionLocation=2998b885bdef,39583,1731483206286 2024-11-13T07:33:43,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,39583,1731483206286 because future has completed 2024-11-13T07:33:43,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T07:33:43,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,39583,1731483206286 in 193 msec 2024-11-13T07:33:43,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE in 591 msec 2024-11-13T07:33:43,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:33:43,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:33:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:55464 deadline: 1731483283028, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=39583 startCode=1731483206286. As of locationSeqNum=5. 2024-11-13T07:33:43,033 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=39583 startCode=1731483206286. As of locationSeqNum=5. 2024-11-13T07:33:43,034 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=39583 startCode=1731483206286. As of locationSeqNum=5. 2024-11-13T07:33:43,034 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,39583,1731483206286, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=39583 startCode=1731483206286. As of locationSeqNum=5. 2024-11-13T07:33:43,142 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:33:43,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:33:43,158 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47901928f170b169c8ff6f7dea663386 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T07:33:43,175 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/9f81e345364940129eed464c65f450c7 is 29, key is r1/cf1:/1731483223146/DeleteFamily/seqid=0 2024-11-13T07:33:43,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741888_1066 (size=4906) 2024-11-13T07:33:43,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741888_1066 (size=4906) 2024-11-13T07:33:43,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741888_1066 (size=4906) 2024-11-13T07:33:43,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,188 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,202 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 is 29, key is r1/cf2:/1731483223146/DeleteFamily/seqid=0 2024-11-13T07:33:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741889_1067 (size=4906) 2024-11-13T07:33:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741889_1067 (size=4906) 2024-11-13T07:33:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741889_1067 (size=4906) 2024-11-13T07:33:43,209 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,215 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,216 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/9f81e345364940129eed464c65f450c7 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,223 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,223 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T07:33:43,225 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,231 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,231 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T07:33:43,232 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 47901928f170b169c8ff6f7dea663386 in 74ms, sequenceid=12, compaction requested=false 2024-11-13T07:33:43,232 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 47901928f170b169c8ff6f7dea663386: 2024-11-13T07:33:43,235 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T07:33:43,237 DEBUG [Time-limited test {}] regionserver.HStore(1541): 47901928f170b169c8ff6f7dea663386/cf1 is initiating major compaction (all files) 2024-11-13T07:33:43,237 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:33:43,237 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:43,238 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 47901928f170b169c8ff6f7dea663386/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,238 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6, hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7] into tmpdir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp, totalSize=9.5 K 2024-11-13T07:33:43,240 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d002e675e2cd4cb08f956a316f72a8e6, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731483222400 2024-11-13T07:33:43,240 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9f81e345364940129eed464c65f450c7, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T07:33:43,252 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 47901928f170b169c8ff6f7dea663386#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T07:33:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741890_1068 (size=4626) 2024-11-13T07:33:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741890_1068 (size=4626) 2024-11-13T07:33:43,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741890_1068 (size=4626) 2024-11-13T07:33:43,267 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf1/6fac5d7cd0af48a893129021ba528693 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/6fac5d7cd0af48a893129021ba528693 2024-11-13T07:33:43,281 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 47901928f170b169c8ff6f7dea663386/cf1 of 47901928f170b169c8ff6f7dea663386 into 6fac5d7cd0af48a893129021ba528693(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T07:33:43,281 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 47901928f170b169c8ff6f7dea663386: 2024-11-13T07:33:43,281 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T07:33:43,281 DEBUG [Time-limited test {}] regionserver.HStore(1541): 47901928f170b169c8ff6f7dea663386/cf2 is initiating major compaction (all files) 2024-11-13T07:33:43,281 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:33:43,281 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:33:43,282 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 47901928f170b169c8ff6f7dea663386/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,282 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5] into tmpdir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp, totalSize=4.8 K 2024-11-13T07:33:43,282 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fd2dd9acd1e649628ed0bef7d9eb95a5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T07:33:43,289 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 47901928f170b169c8ff6f7dea663386#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T07:33:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741891_1069 (size=4592) 2024-11-13T07:33:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741891_1069 (size=4592) 2024-11-13T07:33:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741891_1069 (size=4592) 2024-11-13T07:33:43,305 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/.tmp/cf2/dc7cad71a8d84453873aebfd3874c3a8 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/dc7cad71a8d84453873aebfd3874c3a8 2024-11-13T07:33:43,312 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 47901928f170b169c8ff6f7dea663386/cf2 of 47901928f170b169c8ff6f7dea663386 into dc7cad71a8d84453873aebfd3874c3a8(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T07:33:43,312 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 47901928f170b169c8ff6f7dea663386: 2024-11-13T07:33:43,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=47901928f170b169c8ff6f7dea663386, source=2998b885bdef,39583,1731483206286, destination=2998b885bdef,38607,1731483206428, warming up region on 2998b885bdef,38607,1731483206428 2024-11-13T07:33:43,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=47901928f170b169c8ff6f7dea663386, source=2998b885bdef,39583,1731483206286, destination=2998b885bdef,38607,1731483206428, running balancer 2024-11-13T07:33:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE 2024-11-13T07:33:43,319 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE 2024-11-13T07:33:43,321 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=CLOSING, regionLocation=2998b885bdef,39583,1731483206286 2024-11-13T07:33:43,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(7855): Warmup {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:43,322 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,323 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf1 2024-11-13T07:33:43,323 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:43,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE because future has completed 2024-11-13T07:33:43,325 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T07:33:43,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,39583,1731483206286}] 2024-11-13T07:33:43,333 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/6fac5d7cd0af48a893129021ba528693 2024-11-13T07:33:43,338 INFO [StoreFileOpener-47901928f170b169c8ff6f7dea663386-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,339 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,345 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6 2024-11-13T07:33:43,345 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:43,345 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,346 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf2 2024-11-13T07:33:43,346 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:43,353 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/dc7cad71a8d84453873aebfd3874c3a8 2024-11-13T07:33:43,358 INFO [StoreFileOpener-47901928f170b169c8ff6f7dea663386-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,358 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,358 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:33:43,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:33:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38607 {}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483223359Disabling compacts and flushes for region at 1731483223359Disabling writes for close at 1731483223359Writing region close event to WAL at 1731483223360 (+1 ms)Closed at 1731483223360 2024-11-13T07:33:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T07:33:43,479 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,479 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T07:33:43,480 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:33:43,480 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,480 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,480 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:33:43,480 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,481 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6, hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7] to archive 2024-11-13T07:33:43,484 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T07:33:43,488 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6 to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/d002e675e2cd4cb08f956a316f72a8e6 2024-11-13T07:33:43,490 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7 to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/9f81e345364940129eed464c65f450c7 2024-11-13T07:33:43,502 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5] to archive 2024-11-13T07:33:43,503 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T07:33:43,504 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/fd2dd9acd1e649628ed0bef7d9eb95a5 2024-11-13T07:33:43,509 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T07:33:43,510 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,510 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483223480Running coprocessor pre-close hooks at 1731483223480Disabling compacts and flushes for region at 1731483223480Disabling writes for close at 1731483223480Writing region close event to WAL at 1731483223506 (+26 ms)Running coprocessor post-close hooks at 1731483223510 (+4 ms)Closed at 1731483223510 2024-11-13T07:33:43,510 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 47901928f170b169c8ff6f7dea663386 move to 2998b885bdef,38607,1731483206428 record at close sequenceid=12 2024-11-13T07:33:43,512 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=CLOSED 2024-11-13T07:33:43,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,39583,1731483206286 because future has completed 2024-11-13T07:33:43,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T07:33:43,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,39583,1731483206286 in 191 msec 2024-11-13T07:33:43,520 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE; state=CLOSED, location=2998b885bdef,38607,1731483206428; forceNewPlan=false, retain=false 2024-11-13T07:33:43,670 INFO [2998b885bdef:44967 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T07:33:43,671 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPENING, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:43,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE because future has completed 2024-11-13T07:33:43,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428}] 2024-11-13T07:33:43,838 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,839 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:43,839 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,840 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:43,840 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,840 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,841 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,843 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf1 2024-11-13T07:33:43,843 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:43,849 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/6fac5d7cd0af48a893129021ba528693 2024-11-13T07:33:43,849 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:43,849 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,850 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf2 2024-11-13T07:33:43,851 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:43,857 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/dc7cad71a8d84453873aebfd3874c3a8 2024-11-13T07:33:43,857 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:43,857 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,858 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,859 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,860 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,860 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,861 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T07:33:43,863 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,864 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 47901928f170b169c8ff6f7dea663386; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61038002, jitterRate=-0.09046289324760437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T07:33:43,864 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,864 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 47901928f170b169c8ff6f7dea663386: Running coprocessor pre-open hook at 1731483223840Writing region info on filesystem at 1731483223840Initializing all the Stores at 1731483223841 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483223841Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483223841Cleaning up temporary data from old regions at 1731483223860 (+19 ms)Running coprocessor post-open hooks at 1731483223864 (+4 ms)Region opened successfully at 1731483223864 2024-11-13T07:33:43,865 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., pid=12, masterSystemTime=1731483223829 2024-11-13T07:33:43,868 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,868 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,869 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPEN, openSeqNum=18, regionLocation=2998b885bdef,38607,1731483206428 2024-11-13T07:33:43,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428 because future has completed 2024-11-13T07:33:43,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T07:33:43,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,38607,1731483206428 in 196 msec 2024-11-13T07:33:43,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, REOPEN/MOVE in 558 msec 2024-11-13T07:33:43,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:33:43,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:33:43,924 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2998b885bdef,38607,1731483206428: testing ***** 2024-11-13T07:33:43,925 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T07:33:43,926 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T07:33:43,928 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T07:33:43,930 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T07:33:43,931 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T07:33:43,939 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 386170600 }, "NonHeapMemoryUsage": { "committed": 171442176, "init": 7667712, "max": -1, "used": 168823352 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2998b885bdef", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2998b885bdef", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:65209", "tag.serverName": "2998b885bdef,39583,1731483206286", "tag.clusterId": "24009036-1b1f-487a-9cff-0cd7f006815f", "tag.Context": "regionserver", "tag.Hostname": "2998b885bdef", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731483206286, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T07:33:43,943 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44967 {}] master.MasterRpcServices(700): 2998b885bdef,38607,1731483206428 reported a fatal error: ***** ABORTING region server 2998b885bdef,38607,1731483206428: testing ***** 2024-11-13T07:33:43,945 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,38607,1731483206428' ***** 2024-11-13T07:33:43,945 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T07:33:43,945 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:33:43,945 INFO [RS:2;2998b885bdef:38607 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T07:33:43,945 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:33:43,946 INFO [RS:2;2998b885bdef:38607 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T07:33:43,946 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(3091): Received CLOSE for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,946 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(956): aborting server 2998b885bdef,38607,1731483206428 2024-11-13T07:33:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39583 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:36086 deadline: 1731483283946, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=38607 startCode=1731483206428. As of locationSeqNum=12. 2024-11-13T07:33:43,946 INFO [RS:2;2998b885bdef:38607 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:33:43,947 INFO [RS:2;2998b885bdef:38607 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2998b885bdef:38607. 2024-11-13T07:33:43,947 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:33:43,947 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,947 DEBUG [RS:2;2998b885bdef:38607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:33:43,947 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,947 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:33:43,947 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,947 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,39583,1731483206286, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,39583,1731483206286, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=38607 startCode=1731483206428. As of locationSeqNum=12. 2024-11-13T07:33:43,947 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,39583,1731483206286, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=38607 startCode=1731483206428. As of locationSeqNum=12. 2024-11-13T07:33:43,947 DEBUG [RS:2;2998b885bdef:38607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:33:43,947 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,39583,1731483206286, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2998b885bdef port=38607 startCode=1731483206428. As of locationSeqNum=12. 2024-11-13T07:33:43,948 INFO [RS:2;2998b885bdef:38607 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:33:43,948 INFO [RS:2;2998b885bdef:38607 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:33:43,948 INFO [RS:2;2998b885bdef:38607 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:33:43,948 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T07:33:43,949 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T07:33:43,949 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1325): Online Regions={47901928f170b169c8ff6f7dea663386=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., 1588230740=hbase:meta,,1.1588230740} 2024-11-13T07:33:43,949 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:33:43,949 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:33:43,949 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:33:43,949 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:33:43,949 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:33:43,949 DEBUG [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:43,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2998b885bdef,38607,1731483206428 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:43,950 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2998b885bdef,38607,1731483206428 aborting 2024-11-13T07:33:43,950 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2998b885bdef,38607,1731483206428 aborting 2024-11-13T07:33:43,950 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 from cache 2024-11-13T07:33:43,951 ERROR [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-13T07:33:43,951 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T07:33:43,951 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:33:43,951 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,951 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731483223949Running coprocessor pre-close hooks at 1731483223949Disabling compacts and flushes for region at 1731483223949Disabling writes for close at 1731483223949Writing region close event to WAL at 1731483223951 (+2 ms)Running coprocessor post-close hooks at 1731483223951Closed at 1731483223951 2024-11-13T07:33:43,951 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483223946Running coprocessor pre-close hooks at 1731483223947 (+1 ms)Disabling compacts and flushes for region at 1731483223947Disabling writes for close at 1731483223947Writing region close event to WAL at 1731483223951 (+4 ms)Running coprocessor post-close hooks at 1731483223951Closed at 1731483223951 2024-11-13T07:33:43,951 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:43,951 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T07:33:43,956 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T07:33:43,956 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T07:33:43,999 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:33:44,061 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:44,063 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1] 2024-11-13T07:33:44,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2998b885bdef,38607,1731483206428 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:44,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2998b885bdef,38607,1731483206428 aborting 2024-11-13T07:33:44,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2998b885bdef,38607,1731483206428 aborting 2024-11-13T07:33:44,064 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 from cache 2024-11-13T07:33:44,149 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,38607,1731483206428; all regions closed. 2024-11-13T07:33:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741836_1012 (size=7323) 2024-11-13T07:33:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741836_1012 (size=7323) 2024-11-13T07:33:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741836_1012 (size=7323) 2024-11-13T07:33:44,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741835_1011 (size=2288) 2024-11-13T07:33:44,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741835_1011 (size=2288) 2024-11-13T07:33:44,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741835_1011 (size=2288) 2024-11-13T07:33:44,157 DEBUG [RS:2;2998b885bdef:38607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:33:44,157 INFO [RS:2;2998b885bdef:38607 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:33:44,157 INFO [RS:2;2998b885bdef:38607 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:33:44,157 INFO [RS:2;2998b885bdef:38607 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T07:33:44,157 INFO [RS:2;2998b885bdef:38607 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:33:44,157 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:33:44,158 INFO [RS:2;2998b885bdef:38607 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38607 2024-11-13T07:33:44,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,38607,1731483206428 2024-11-13T07:33:44,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:33:44,225 INFO [RS:2;2998b885bdef:38607 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:33:44,236 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,38607,1731483206428] 2024-11-13T07:33:44,246 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,38607,1731483206428 already deleted, retry=false 2024-11-13T07:33:44,247 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2998b885bdef,38607,1731483206428 on 2998b885bdef,44967,1731483205524 2024-11-13T07:33:44,253 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2998b885bdef,38607,1731483206428, splitWal=true, meta=true 2024-11-13T07:33:44,255 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2998b885bdef,38607,1731483206428, splitWal=true, meta=true 2024-11-13T07:33:44,255 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 2998b885bdef,38607,1731483206428 (carryingMeta=true) 2998b885bdef,38607,1731483206428/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4280c1ff[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T07:33:44,257 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2998b885bdef,38607,1731483206428, splitWal=true, meta=true, isMeta: true 2024-11-13T07:33:44,258 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting 2024-11-13T07:33:44,259 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2998b885bdef,38607,1731483206428 WAL count=1, meta=true 2024-11-13T07:33:44,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta}] 2024-11-13T07:33:44,268 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2998b885bdef,37319,1731483206386 2024-11-13T07:33:44,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, worker=2998b885bdef,37319,1731483206386}] 2024-11-13T07:33:44,271 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:44,273 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1] 2024-11-13T07:33:44,274 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2998b885bdef:38607 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2998b885bdef/172.17.0.3:38607 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:44,275 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1, error=java.net.ConnectException: Call to address=2998b885bdef:38607 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2998b885bdef/172.17.0.3:38607 2024-11-13T07:33:44,275 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-13T07:33:44,275 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 from cache 2024-11-13T07:33:44,275 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 2998b885bdef:38607 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2998b885bdef/172.17.0.3:38607 2024-11-13T07:33:44,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:33:44,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x101331cf8110003, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:33:44,337 INFO [RS:2;2998b885bdef:38607 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:33:44,337 INFO [RS:2;2998b885bdef:38607 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,38607,1731483206428; zookeeper connection closed. 2024-11-13T07:33:44,337 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16b2d297 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16b2d297 2024-11-13T07:33:44,430 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:33:44,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58761, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:33:44,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37319 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T07:33:44,452 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, size=7.2 K (7323bytes) 2024-11-13T07:33:44,452 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta 2024-11-13T07:33:44,452 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta after 0ms 2024-11-13T07:33:44,475 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:44,476 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta took 25ms 2024-11-13T07:33:44,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-13T07:33:44,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta so closing down 2024-11-13T07:33:44,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:44,484 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp 2024-11-13T07:33:44,508 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp 2024-11-13T07:33:44,508 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:44,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741892_1070 (size=7071) 2024-11-13T07:33:44,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741892_1070 (size=7071) 2024-11-13T07:33:44,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741892_1070 (size=7071) 2024-11-13T07:33:44,560 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:44,562 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T07:33:44,562 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 86 ms; skipped=1; WAL=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-13T07:33:44,562 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, journal: Splitting hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, size=7.2 K (7323bytes) at 1731483224452Finishing writing output for hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta so closing down at 1731483224482 (+30 ms)Creating recovered edits writer path=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp at 1731483224508 (+26 ms)3 split writer threads finished at 1731483224508Closed recovered edits writer path=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731483224560 (+52 ms)Rename recovered edits hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta.temp to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731483224562 (+2 ms)Processed 16 edits across 1 Regions in 86 ms; skipped=1; WAL=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1731483224562 2024-11-13T07:33:44,562 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta 2024-11-13T07:33:44,563 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T07:33:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T07:33:44,570 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs 2024-11-13T07:33:44,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T07:33:44,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, worker=2998b885bdef,37319,1731483206386 in 299 msec 2024-11-13T07:33:44,574 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=2998b885bdef,37319,1731483206386 2024-11-13T07:33:44,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T07:33:44,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2998b885bdef%2C38607%2C1731483206428.meta.1731483208505.meta, worker=2998b885bdef,37319,1731483206386 in 314 msec 2024-11-13T07:33:44,579 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting dir is empty, no logs to split. 2024-11-13T07:33:44,579 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2998b885bdef,38607,1731483206428 WAL count=0, meta=true 2024-11-13T07:33:44,579 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 2998b885bdef,38607,1731483206428 WAL splitting is done? wals=0, meta=true 2024-11-13T07:33:44,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T07:33:44,581 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:44,582 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:33:44,583 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1] 2024-11-13T07:33:44,583 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:33:44,583 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2998b885bdef:38607 this server is in the failed servers list 2024-11-13T07:33:44,584 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2998b885bdef:38607 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2998b885bdef:38607 2024-11-13T07:33:44,584 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2998b885bdef:38607 2024-11-13T07:33:44,584 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 from cache 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:33:44,734 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:33:44,734 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:33:44,734 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:33:44,735 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2998b885bdef,39583,1731483206286 2024-11-13T07:33:44,737 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,39583,1731483206286, state=OPENING 2024-11-13T07:33:44,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:44,757 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:44,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:44,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:44,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:44,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:44,758 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:33:44,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,39583,1731483206286}] 2024-11-13T07:33:44,924 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T07:33:44,924 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:44,925 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T07:33:44,927 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C39583%2C1731483206286.meta, suffix=.meta, logDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286, archiveDir=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs, maxLogs=32 2024-11-13T07:33:44,942 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286/2998b885bdef%2C39583%2C1731483206286.meta.1731483224928.meta, exclude list is [], retry=0 2024-11-13T07:33:44,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:44,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:44,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:44,970 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,39583,1731483206286/2998b885bdef%2C39583%2C1731483206286.meta.1731483224928.meta 2024-11-13T07:33:44,970 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:44,970 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:44,971 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T07:33:44,971 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T07:33:44,971 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T07:33:44,971 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T07:33:44,971 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:44,971 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T07:33:44,971 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T07:33:44,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:33:44,973 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:33:44,973 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:44,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:44,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:33:44,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:33:44,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:44,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:44,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:33:44,976 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:33:44,976 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:44,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:44,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:33:44,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:33:44,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:44,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:33:44,979 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:33:44,979 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740 2024-11-13T07:33:44,981 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740 2024-11-13T07:33:44,982 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T07:33:45,008 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:45,011 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T07:33:45,012 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-13T07:33:45,030 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/info/4ec5d890d9e34e08a6d6a2a2fb35423e is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386./info:regioninfo/1731483223869/Put/seqid=0 2024-11-13T07:33:45,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741894_1072 (size=11177) 2024-11-13T07:33:45,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741894_1072 (size=11177) 2024-11-13T07:33:45,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741894_1072 (size=11177) 2024-11-13T07:33:45,037 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/info/4ec5d890d9e34e08a6d6a2a2fb35423e 2024-11-13T07:33:45,056 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/ns/f97048d0da564464a925eb215a11e382 is 43, key is default/ns:d/1731483208735/Put/seqid=0 2024-11-13T07:33:45,092 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:45,094 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1] 2024-11-13T07:33:45,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39583 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.3:36086 deadline: 1731483285095, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 2998b885bdef,39583,1731483206286 2024-11-13T07:33:45,096 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 2998b885bdef,39583,1731483206286 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T07:33:45,097 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 2998b885bdef,39583,1731483206286 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T07:33:45,097 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1 from cache 2024-11-13T07:33:45,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741895_1073 (size=5153) 2024-11-13T07:33:45,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741895_1073 (size=5153) 2024-11-13T07:33:45,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741895_1073 (size=5153) 2024-11-13T07:33:45,207 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/ns/f97048d0da564464a925eb215a11e382 2024-11-13T07:33:45,230 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/table/3d1c6c83c75d4ca6adf099125452e1b4 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731483222154/Put/seqid=0 2024-11-13T07:33:45,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741896_1074 (size=5431) 2024-11-13T07:33:45,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741896_1074 (size=5431) 2024-11-13T07:33:45,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741896_1074 (size=5431) 2024-11-13T07:33:45,237 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/table/3d1c6c83c75d4ca6adf099125452e1b4 2024-11-13T07:33:45,244 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/info/4ec5d890d9e34e08a6d6a2a2fb35423e as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/info/4ec5d890d9e34e08a6d6a2a2fb35423e 2024-11-13T07:33:45,251 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/info/4ec5d890d9e34e08a6d6a2a2fb35423e, entries=36, sequenceid=18, filesize=10.9 K 2024-11-13T07:33:45,252 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/ns/f97048d0da564464a925eb215a11e382 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/ns/f97048d0da564464a925eb215a11e382 2024-11-13T07:33:45,259 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/ns/f97048d0da564464a925eb215a11e382, entries=2, sequenceid=18, filesize=5.0 K 2024-11-13T07:33:45,260 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/table/3d1c6c83c75d4ca6adf099125452e1b4 as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/table/3d1c6c83c75d4ca6adf099125452e1b4 2024-11-13T07:33:45,265 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/table/3d1c6c83c75d4ca6adf099125452e1b4, entries=2, sequenceid=18, filesize=5.3 K 2024-11-13T07:33:45,265 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 253ms, sequenceid=18, compaction requested=false; wal=null 2024-11-13T07:33:45,266 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T07:33:45,268 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:33:45,268 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:33:45,268 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:33:45,270 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:33:45,272 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T07:33:45,273 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74908875, jitterRate=0.11622922122478485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:33:45,273 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T07:33:45,274 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731483224971Writing region info on filesystem at 1731483224971Initializing all the Stores at 1731483224972 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483224972Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483224972Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483224972Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731483224972Obtaining lock to block concurrent updates at 1731483225012 (+40 ms)Preparing flush snapshotting stores in 1588230740 at 1731483225012Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731483225012Flushing stores of hbase:meta,,1.1588230740 at 1731483225012Flushing 1588230740/info: creating writer at 1731483225012Flushing 1588230740/info: appending metadata at 1731483225030 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731483225030Flushing 1588230740/ns: creating writer at 1731483225043 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731483225056 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731483225056Flushing 1588230740/table: creating writer at 1731483225217 (+161 ms)Flushing 1588230740/table: appending metadata at 1731483225230 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731483225230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e4f0887: reopening flushed file at 1731483225243 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fc888a7: reopening flushed file at 1731483225251 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54bb0892: reopening flushed file at 1731483225259 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 253ms, sequenceid=18, compaction requested=false; wal=null at 1731483225265 (+6 ms)Cleaning up temporary data from old regions at 1731483225268 (+3 ms)Running coprocessor post-open hooks at 1731483225273 (+5 ms)Region opened successfully at 1731483225274 (+1 ms) 2024-11-13T07:33:45,275 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731483224913 2024-11-13T07:33:45,277 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T07:33:45,277 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T07:33:45,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=2998b885bdef,39583,1731483206286 2024-11-13T07:33:45,279 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,39583,1731483206286, state=OPEN 2024-11-13T07:33:45,327 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44967 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,39583,1731483206286, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:33:45,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:45,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:45,384 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:33:45,384 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2998b885bdef,39583,1731483206286 2024-11-13T07:33:45,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:45,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:45,384 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:33:45,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T07:33:45,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,39583,1731483206286 in 626 msec 2024-11-13T07:33:45,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T07:33:45,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 808 msec 2024-11-13T07:33:45,393 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2998b885bdef,38607,1731483206428 had 2 regions 2024-11-13T07:33:45,394 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2998b885bdef,38607,1731483206428, splitWal=true, meta=true, isMeta: false 2024-11-13T07:33:45,397 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2998b885bdef,38607,1731483206428 WAL count=1, meta=false 2024-11-13T07:33:45,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2998b885bdef%2C38607%2C1731483206428.1731483208030}] 2024-11-13T07:33:45,399 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=2998b885bdef,37319,1731483206386 2024-11-13T07:33:45,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C38607%2C1731483206428.1731483208030, worker=2998b885bdef,37319,1731483206386}] 2024-11-13T07:33:45,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37319 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-13T07:33:45,574 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030, size=2.2 K (2288bytes) 2024-11-13T07:33:45,574 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 2024-11-13T07:33:45,575 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 after 1ms 2024-11-13T07:33:45,599 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:45,599 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 took 25ms 2024-11-13T07:33:45,601 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 47901928f170b169c8ff6f7dea663386: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T07:33:45,602 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 so closing down 2024-11-13T07:33:45,602 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:45,602 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:45,602 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-13T07:33:45,602 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030, journal: Splitting hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030, size=2.2 K (2288bytes) at 1731483225574Finishing writing output for hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 so closing down at 1731483225602 (+28 ms)3 split writer threads finished at 1731483225602Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731483225602 2024-11-13T07:33:45,602 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 2024-11-13T07:33:45,602 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-13T07:33:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44967 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-13T07:33:45,606 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting/2998b885bdef%2C38607%2C1731483206428.1731483208030 to hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs 2024-11-13T07:33:45,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-13T07:33:45,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C38607%2C1731483206428.1731483208030, worker=2998b885bdef,37319,1731483206386 in 207 msec 2024-11-13T07:33:45,610 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=2998b885bdef,37319,1731483206386 2024-11-13T07:33:45,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-13T07:33:45,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2998b885bdef%2C38607%2C1731483206428.1731483208030, worker=2998b885bdef,37319,1731483206386 in 213 msec 2024-11-13T07:33:45,614 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/WALs/2998b885bdef,38607,1731483206428-splitting dir is empty, no logs to split. 2024-11-13T07:33:45,614 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2998b885bdef,38607,1731483206428 WAL count=0, meta=false 2024-11-13T07:33:45,614 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2998b885bdef,38607,1731483206428 WAL splitting is done? wals=0, meta=false 2024-11-13T07:33:45,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN}] 2024-11-13T07:33:45,618 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN 2024-11-13T07:33:45,618 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:33:45,769 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:33:45,770 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:33:45,770 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:33:45,770 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:33:45,770 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:33:45,770 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:33:45,770 INFO [2998b885bdef:44967 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:33:45,770 DEBUG [2998b885bdef:44967 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:33:45,771 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPENING, regionLocation=2998b885bdef,37319,1731483206386 2024-11-13T07:33:45,773 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2998b885bdef:38607 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:38607 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:45,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1, error=java.net.ConnectException: Call to address=2998b885bdef:38607 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:38607 2024-11-13T07:33:45,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T07:33:45,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,38607,1731483206428, seqNum=-1 from cache 2024-11-13T07:33:45,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 2998b885bdef:38607 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:38607 2024-11-13T07:33:45,857 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T07:33:45,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:45,861 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T07:33:45,861 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:45,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:45,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1] 2024-11-13T07:33:45,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:33:45,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56551, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:33:45,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN because future has completed 2024-11-13T07:33:45,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,37319,1731483206386}] 2024-11-13T07:33:46,048 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:46,048 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 47901928f170b169c8ff6f7dea663386, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:46,049 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,049 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:46,049 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,050 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,052 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,052 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf1 2024-11-13T07:33:46,053 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,058 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf1/6fac5d7cd0af48a893129021ba528693 2024-11-13T07:33:46,058 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,059 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,060 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47901928f170b169c8ff6f7dea663386 columnFamilyName cf2 2024-11-13T07:33:46,060 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,066 DEBUG [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/cf2/dc7cad71a8d84453873aebfd3874c3a8 2024-11-13T07:33:46,066 INFO [StoreOpener-47901928f170b169c8ff6f7dea663386-1 {}] regionserver.HStore(327): Store=47901928f170b169c8ff6f7dea663386/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,066 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,067 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,068 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,069 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,069 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,070 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T07:33:46,071 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,072 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 47901928f170b169c8ff6f7dea663386; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66106892, jitterRate=-0.014930546283721924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T07:33:46,072 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:33:46,073 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 47901928f170b169c8ff6f7dea663386: Running coprocessor pre-open hook at 1731483226050Writing region info on filesystem at 1731483226050Initializing all the Stores at 1731483226051 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226051Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226051Cleaning up temporary data from old regions at 1731483226069 (+18 ms)Running coprocessor post-open hooks at 1731483226072 (+3 ms)Region opened successfully at 1731483226072 2024-11-13T07:33:46,074 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., pid=21, masterSystemTime=1731483226039 2024-11-13T07:33:46,076 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:46,076 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:33:46,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=47901928f170b169c8ff6f7dea663386, regionState=OPEN, openSeqNum=18, regionLocation=2998b885bdef,37319,1731483206386 2024-11-13T07:33:46,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,37319,1731483206386 because future has completed 2024-11-13T07:33:46,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T07:33:46,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 47901928f170b169c8ff6f7dea663386, server=2998b885bdef,37319,1731483206386 in 195 msec 2024-11-13T07:33:46,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-13T07:33:46,085 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 2998b885bdef,38607,1731483206428 after splitting done 2024-11-13T07:33:46,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=47901928f170b169c8ff6f7dea663386, ASSIGN in 466 msec 2024-11-13T07:33:46,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 2998b885bdef,38607,1731483206428, splitWal=true, meta=true in 1.8370 sec 2024-11-13T07:33:46,111 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:33:46,113 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,39583,1731483206286, seqNum=-1] 2024-11-13T07:33:46,232 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2998b885bdef:38607 this server is in the failed servers list 2024-11-13T07:33:46,232 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2998b885bdef:38607 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2998b885bdef:38607 2024-11-13T07:33:46,232 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2998b885bdef:38607 2024-11-13T07:33:46,232 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,38607,1731483206428, seqNum=12 from cache 2024-11-13T07:33:46,452 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386., hostname=2998b885bdef,37319,1731483206386, seqNum=18] 2024-11-13T07:33:46,453 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:33:46,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:33:46,468 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 410), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=8419 (was 8458) 2024-11-13T07:33:46,469 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-13T07:33:46,482 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=8418 2024-11-13T07:33:46,482 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-13T07:33:46,501 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:46,503 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:46,503 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:46,506 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-03674719, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-03674719, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:46,517 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-03674719/hregion-03674719.1731483226506, exclude list is [], retry=0 2024-11-13T07:33:46,519 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:46,519 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:46,520 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:46,545 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-03674719/hregion-03674719.1731483226506 2024-11-13T07:33:46,546 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:46,546 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9c4d0284e2bb0583c957c0cb87a29a5b, NAME => 'testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:46,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741898_1076 (size=67) 2024-11-13T07:33:46,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741898_1076 (size=67) 2024-11-13T07:33:46,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741898_1076 (size=67) 2024-11-13T07:33:46,555 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:46,556 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,558 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName a 2024-11-13T07:33:46,558 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,558 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,558 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,559 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName b 2024-11-13T07:33:46,559 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,560 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,560 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,561 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName c 2024-11-13T07:33:46,561 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,561 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,562 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,564 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,564 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,565 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:46,566 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,568 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:46,569 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c4d0284e2bb0583c957c0cb87a29a5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66557720, jitterRate=-0.008212685585021973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:46,569 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c4d0284e2bb0583c957c0cb87a29a5b: Writing region info on filesystem at 1731483226555Initializing all the Stores at 1731483226556 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226556Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226556Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226556Cleaning up temporary data from old regions at 1731483226564 (+8 ms)Region opened successfully at 1731483226569 (+5 ms) 2024-11-13T07:33:46,569 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9c4d0284e2bb0583c957c0cb87a29a5b, disabling compactions & flushes 2024-11-13T07:33:46,569 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,569 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,569 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. after waiting 0 ms 2024-11-13T07:33:46,569 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,570 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,570 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9c4d0284e2bb0583c957c0cb87a29a5b: Waiting for close lock at 1731483226569Disabling compacts and flushes for region at 1731483226569Disabling writes for close at 1731483226569Writing region close event to WAL at 1731483226570 (+1 ms)Closed at 1731483226570 2024-11-13T07:33:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741897_1075 (size=175) 2024-11-13T07:33:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741897_1075 (size=175) 2024-11-13T07:33:46,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741897_1075 (size=175) 2024-11-13T07:33:46,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:46,575 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-03674719:(num 1731483226506) 2024-11-13T07:33:46,575 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:46,576 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:46,587 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577, exclude list is [], retry=0 2024-11-13T07:33:46,589 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:46,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:46,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:46,615 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 2024-11-13T07:33:46,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:46,616 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c4d0284e2bb0583c957c0cb87a29a5b, NAME => 'testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:46,616 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:46,616 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,616 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,618 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,619 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName a 2024-11-13T07:33:46,619 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,619 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,619 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,620 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName b 2024-11-13T07:33:46,620 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,620 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,620 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,621 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName c 2024-11-13T07:33:46,621 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:46,622 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:46,622 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,622 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,624 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,625 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,625 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,625 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:46,627 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,627 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c4d0284e2bb0583c957c0cb87a29a5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71506831, jitterRate=0.0655348151922226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:46,628 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c4d0284e2bb0583c957c0cb87a29a5b: Writing region info on filesystem at 1731483226616Initializing all the Stores at 1731483226617 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226617Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226618 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226618Cleaning up temporary data from old regions at 1731483226625 (+7 ms)Region opened successfully at 1731483226628 (+3 ms) 2024-11-13T07:33:46,653 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9c4d0284e2bb0583c957c0cb87a29a5b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T07:33:46,672 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/a/7b60bea2e8984643941cc0e2c7429288 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731483226628/Put/seqid=0 2024-11-13T07:33:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741900_1078 (size=5958) 2024-11-13T07:33:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741900_1078 (size=5958) 2024-11-13T07:33:46,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741900_1078 (size=5958) 2024-11-13T07:33:46,679 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/a/7b60bea2e8984643941cc0e2c7429288 2024-11-13T07:33:46,697 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/b/fa5ffa6417e04a848d56ed5541aab0c2 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731483226637/Put/seqid=0 2024-11-13T07:33:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741901_1079 (size=5958) 2024-11-13T07:33:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741901_1079 (size=5958) 2024-11-13T07:33:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741901_1079 (size=5958) 2024-11-13T07:33:46,704 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/b/fa5ffa6417e04a848d56ed5541aab0c2 2024-11-13T07:33:46,722 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/c/ba92e672cef04619a0655c3aa1a8b5c0 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731483226644/Put/seqid=0 2024-11-13T07:33:46,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741902_1080 (size=5958) 2024-11-13T07:33:46,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741902_1080 (size=5958) 2024-11-13T07:33:46,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741902_1080 (size=5958) 2024-11-13T07:33:46,729 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/c/ba92e672cef04619a0655c3aa1a8b5c0 2024-11-13T07:33:46,735 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/a/7b60bea2e8984643941cc0e2c7429288 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/a/7b60bea2e8984643941cc0e2c7429288 2024-11-13T07:33:46,740 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/a/7b60bea2e8984643941cc0e2c7429288, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T07:33:46,741 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/b/fa5ffa6417e04a848d56ed5541aab0c2 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/b/fa5ffa6417e04a848d56ed5541aab0c2 2024-11-13T07:33:46,746 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/b/fa5ffa6417e04a848d56ed5541aab0c2, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T07:33:46,747 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/c/ba92e672cef04619a0655c3aa1a8b5c0 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/c/ba92e672cef04619a0655c3aa1a8b5c0 2024-11-13T07:33:46,752 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/c/ba92e672cef04619a0655c3aa1a8b5c0, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T07:33:46,754 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9c4d0284e2bb0583c957c0cb87a29a5b in 101ms, sequenceid=33, compaction requested=false 2024-11-13T07:33:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9c4d0284e2bb0583c957c0cb87a29a5b: 2024-11-13T07:33:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9c4d0284e2bb0583c957c0cb87a29a5b, disabling compactions & flushes 2024-11-13T07:33:46,754 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. after waiting 0 ms 2024-11-13T07:33:46,754 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,755 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. 2024-11-13T07:33:46,755 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9c4d0284e2bb0583c957c0cb87a29a5b: Waiting for close lock at 1731483226754Disabling compacts and flushes for region at 1731483226754Disabling writes for close at 1731483226754Writing region close event to WAL at 1731483226755 (+1 ms)Closed at 1731483226755 2024-11-13T07:33:46,759 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 not finished, retry = 0 2024-11-13T07:33:46,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741899_1077 (size=6311) 2024-11-13T07:33:46,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741899_1077 (size=6311) 2024-11-13T07:33:46,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741899_1077 (size=6311) 2024-11-13T07:33:46,863 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/b/fa5ffa6417e04a848d56ed5541aab0c2 to hdfs://localhost:35647/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/b/fa5ffa6417e04a848d56ed5541aab0c2 2024-11-13T07:33:46,880 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577, size=6.2 K (6311bytes) 2024-11-13T07:33:46,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 2024-11-13T07:33:46,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 after 0ms 2024-11-13T07:33:46,905 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:46,906 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 took 26ms 2024-11-13T07:33:46,908 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 so closing down 2024-11-13T07:33:46,908 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:46,909 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731483226577.temp 2024-11-13T07:33:46,931 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000003-wal.1731483226577.temp 2024-11-13T07:33:46,931 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:46,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741903_1081 (size=5575) 2024-11-13T07:33:46,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741903_1081 (size=5575) 2024-11-13T07:33:46,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741903_1081 (size=5575) 2024-11-13T07:33:46,941 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000003-wal.1731483226577.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:46,942 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000003-wal.1731483226577.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000032 2024-11-13T07:33:46,942 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-13T07:33:46,942 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577, journal: Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577, size=6.2 K (6311bytes) at 1731483226880Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 so closing down at 1731483226908 (+28 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000003-wal.1731483226577.temp at 1731483226931 (+23 ms)3 split writer threads finished at 1731483226931Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000003-wal.1731483226577.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731483226941 (+10 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000003-wal.1731483226577.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000032 at 1731483226942 (+1 ms)Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731483226942 2024-11-13T07:33:46,944 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226577 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483226577 2024-11-13T07:33:46,945 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000032 2024-11-13T07:33:46,945 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:46,946 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:46,959 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226947, exclude list is [], retry=0 2024-11-13T07:33:46,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:46,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:46,962 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:46,993 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226947 2024-11-13T07:33:46,993 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:33:46,993 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c4d0284e2bb0583c957c0cb87a29a5b, NAME => 'testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:46,993 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:46,994 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,994 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,995 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:46,996 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName a 2024-11-13T07:33:46,996 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,001 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/a/7b60bea2e8984643941cc0e2c7429288 2024-11-13T07:33:47,002 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,002 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,003 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName b 2024-11-13T07:33:47,003 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,004 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,004 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,005 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c4d0284e2bb0583c957c0cb87a29a5b columnFamilyName c 2024-11-13T07:33:47,005 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,010 DEBUG [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/c/ba92e672cef04619a0655c3aa1a8b5c0 2024-11-13T07:33:47,010 INFO [StoreOpener-9c4d0284e2bb0583c957c0cb87a29a5b-1 {}] regionserver.HStore(327): Store=9c4d0284e2bb0583c957c0cb87a29a5b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,010 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,011 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,012 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,013 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000032 2024-11-13T07:33:47,039 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:47,040 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000032 2024-11-13T07:33:47,040 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9c4d0284e2bb0583c957c0cb87a29a5b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T07:33:47,052 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/b/b3bd5b45a8774c9189dd7c69dbddf24c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731483226637/Put/seqid=0 2024-11-13T07:33:47,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741905_1083 (size=5958) 2024-11-13T07:33:47,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741905_1083 (size=5958) 2024-11-13T07:33:47,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741905_1083 (size=5958) 2024-11-13T07:33:47,060 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/b/b3bd5b45a8774c9189dd7c69dbddf24c 2024-11-13T07:33:47,066 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/.tmp/b/b3bd5b45a8774c9189dd7c69dbddf24c as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/b/b3bd5b45a8774c9189dd7c69dbddf24c 2024-11-13T07:33:47,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/b/b3bd5b45a8774c9189dd7c69dbddf24c, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T07:33:47,072 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9c4d0284e2bb0583c957c0cb87a29a5b in 32ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T07:33:47,073 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/0000000000000000032 2024-11-13T07:33:47,074 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,074 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,075 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9c4d0284e2bb0583c957c0cb87a29a5b 2024-11-13T07:33:47,080 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/9c4d0284e2bb0583c957c0cb87a29a5b/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T07:33:47,081 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9c4d0284e2bb0583c957c0cb87a29a5b; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61979801, jitterRate=-0.07642899453639984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:47,082 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9c4d0284e2bb0583c957c0cb87a29a5b: Writing region info on filesystem at 1731483226994Initializing all the Stores at 1731483226995 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226995Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226995Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483226995Obtaining lock to block concurrent updates at 1731483227040 (+45 ms)Preparing flush snapshotting stores in 9c4d0284e2bb0583c957c0cb87a29a5b at 1731483227040Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731483227040Flushing stores of testReplayEditsWrittenViaHRegion,,1731483226501.9c4d0284e2bb0583c957c0cb87a29a5b. at 1731483227040Flushing 9c4d0284e2bb0583c957c0cb87a29a5b/b: creating writer at 1731483227040Flushing 9c4d0284e2bb0583c957c0cb87a29a5b/b: appending metadata at 1731483227052 (+12 ms)Flushing 9c4d0284e2bb0583c957c0cb87a29a5b/b: closing flushed file at 1731483227052Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@619d8da2: reopening flushed file at 1731483227065 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9c4d0284e2bb0583c957c0cb87a29a5b in 32ms, sequenceid=32, compaction requested=false; wal=null at 1731483227072 (+7 ms)Cleaning up temporary data from old regions at 1731483227074 (+2 ms)Region opened successfully at 1731483227081 (+7 ms) 2024-11-13T07:33:47,105 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 409) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:55856 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:59584 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:46604 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:33828 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:41596 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:56762 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1118 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=8412 (was 8418) 2024-11-13T07:33:47,105 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-13T07:33:47,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1118, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=8411 2024-11-13T07:33:47,118 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1118 is superior to 1024 2024-11-13T07:33:47,138 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:47,141 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:47,142 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:47,145 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12587985, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-12587985, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:47,165 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12587985/hregion-12587985.1731483227146, exclude list is [], retry=0 2024-11-13T07:33:47,168 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:47,168 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:47,168 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:47,195 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12587985/hregion-12587985.1731483227146 2024-11-13T07:33:47,196 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:47,196 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8e5384754a2b7eb8e38b90c2b437394b, NAME => 'testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741907_1085 (size=68) 2024-11-13T07:33:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741907_1085 (size=68) 2024-11-13T07:33:47,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741907_1085 (size=68) 2024-11-13T07:33:47,208 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:47,212 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,213 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName a 2024-11-13T07:33:47,213 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,214 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,214 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,215 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName b 2024-11-13T07:33:47,216 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,216 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,216 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,217 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName c 2024-11-13T07:33:47,218 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,218 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,218 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,220 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,220 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,220 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:47,221 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,223 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:47,224 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e5384754a2b7eb8e38b90c2b437394b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62317013, jitterRate=-0.07140414416790009}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:47,224 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e5384754a2b7eb8e38b90c2b437394b: Writing region info on filesystem at 1731483227208Initializing all the Stores at 1731483227209 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483227209Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483227211 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483227211Cleaning up temporary data from old regions at 1731483227220 (+9 ms)Region opened successfully at 1731483227224 (+4 ms) 2024-11-13T07:33:47,224 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8e5384754a2b7eb8e38b90c2b437394b, disabling compactions & flushes 2024-11-13T07:33:47,224 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:47,224 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:47,224 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. after waiting 0 ms 2024-11-13T07:33:47,224 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:47,225 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:47,225 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8e5384754a2b7eb8e38b90c2b437394b: Waiting for close lock at 1731483227224Disabling compacts and flushes for region at 1731483227224Disabling writes for close at 1731483227224Writing region close event to WAL at 1731483227225 (+1 ms)Closed at 1731483227225 2024-11-13T07:33:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741906_1084 (size=175) 2024-11-13T07:33:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741906_1084 (size=175) 2024-11-13T07:33:47,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741906_1084 (size=175) 2024-11-13T07:33:47,230 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:47,230 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12587985:(num 1731483227146) 2024-11-13T07:33:47,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:47,231 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:47,243 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231, exclude list is [], retry=0 2024-11-13T07:33:47,245 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:47,246 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:47,246 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:47,270 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 2024-11-13T07:33:47,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:47,327 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e5384754a2b7eb8e38b90c2b437394b, NAME => 'testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:47,329 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,329 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:47,329 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,329 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,331 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,332 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName a 2024-11-13T07:33:47,332 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,333 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,333 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,334 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName b 2024-11-13T07:33:47,334 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,335 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,335 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,335 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName c 2024-11-13T07:33:47,335 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:47,336 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:47,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,338 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:47,340 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,341 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e5384754a2b7eb8e38b90c2b437394b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71432194, jitterRate=0.06442263722419739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:47,342 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:47,342 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e5384754a2b7eb8e38b90c2b437394b: Running coprocessor pre-open hook at 1731483227329Writing region info on filesystem at 1731483227329Initializing all the Stores at 1731483227331 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483227331Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483227331Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483227331Cleaning up temporary data from old regions at 1731483227339 (+8 ms)Running coprocessor post-open hooks at 1731483227342 (+3 ms)Region opened successfully at 1731483227342 2024-11-13T07:33:47,354 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e5384754a2b7eb8e38b90c2b437394b 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T07:33:47,355 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:47,666 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:33:48,356 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:49,358 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:50,359 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:51,360 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:52,360 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:52,833 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:33:53,361 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:54,362 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:54,662 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:33:55,363 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:55,857 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T07:33:55,857 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:55,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T07:33:55,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:55,859 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T07:33:55,859 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T07:33:56,363 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8e5384754a2b7eb8e38b90c2b437394b/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:33:56,365 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e5384754a2b7eb8e38b90c2b437394b: 2024-11-13T07:33:56,365 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:56,379 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e5384754a2b7eb8e38b90c2b437394b: 2024-11-13T07:33:56,380 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T07:33:56,380 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8e5384754a2b7eb8e38b90c2b437394b, disabling compactions & flushes 2024-11-13T07:33:56,380 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:56,380 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:56,380 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. after waiting 0 ms 2024-11-13T07:33:56,380 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:56,380 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:56,380 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. 2024-11-13T07:33:56,380 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8e5384754a2b7eb8e38b90c2b437394b: Waiting for close lock at 1731483236380Running coprocessor pre-close hooks at 1731483236380Disabling compacts and flushes for region at 1731483236380Disabling writes for close at 1731483236380Writing region close event to WAL at 1731483236380Running coprocessor post-close hooks at 1731483236380Closed at 1731483236380 2024-11-13T07:33:56,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741908_1086 (size=4236) 2024-11-13T07:33:56,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741908_1086 (size=4236) 2024-11-13T07:33:56,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741908_1086 (size=4236) 2024-11-13T07:33:56,397 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231, size=4.1 K (4236bytes) 2024-11-13T07:33:56,397 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 2024-11-13T07:33:56,397 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 after 0ms 2024-11-13T07:33:56,422 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:56,422 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 took 25ms 2024-11-13T07:33:56,424 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 so closing down 2024-11-13T07:33:56,424 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:56,425 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731483227231.temp 2024-11-13T07:33:56,449 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000004-wal.1731483227231.temp 2024-11-13T07:33:56,450 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:56,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741909_1087 (size=3245) 2024-11-13T07:33:56,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741909_1087 (size=3245) 2024-11-13T07:33:56,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741909_1087 (size=3245) 2024-11-13T07:33:56,456 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000004-wal.1731483227231.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:56,458 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000004-wal.1731483227231.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000026 2024-11-13T07:33:56,458 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-13T07:33:56,458 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231, journal: Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231, size=4.1 K (4236bytes) at 1731483236397Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 so closing down at 1731483236424 (+27 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000004-wal.1731483227231.temp at 1731483236449 (+25 ms)3 split writer threads finished at 1731483236450 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000004-wal.1731483227231.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731483236456 (+6 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000004-wal.1731483227231.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000026 at 1731483236458 (+2 ms)Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731483236458 2024-11-13T07:33:56,460 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483227231 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483227231 2024-11-13T07:33:56,461 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000026 2024-11-13T07:33:56,461 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:56,463 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:56,475 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483236463, exclude list is [], retry=0 2024-11-13T07:33:56,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:56,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:56,479 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:56,508 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483236463 2024-11-13T07:33:56,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:33:56,509 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e5384754a2b7eb8e38b90c2b437394b, NAME => 'testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:56,509 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,510 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:56,510 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,510 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,512 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,512 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName a 2024-11-13T07:33:56,513 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:56,513 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:56,513 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,514 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName b 2024-11-13T07:33:56,514 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:56,514 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:56,515 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,515 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e5384754a2b7eb8e38b90c2b437394b columnFamilyName c 2024-11-13T07:33:56,515 DEBUG [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:56,516 INFO [StoreOpener-8e5384754a2b7eb8e38b90c2b437394b-1 {}] regionserver.HStore(327): Store=8e5384754a2b7eb8e38b90c2b437394b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:56,516 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,518 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,519 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000026 2024-11-13T07:33:56,543 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:56,545 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000026 2024-11-13T07:33:56,545 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e5384754a2b7eb8e38b90c2b437394b 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T07:33:56,559 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/a/f325778e59ef490d9532794d8e538020 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731483236370/Put/seqid=0 2024-11-13T07:33:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741911_1089 (size=5523) 2024-11-13T07:33:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741911_1089 (size=5523) 2024-11-13T07:33:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741911_1089 (size=5523) 2024-11-13T07:33:56,566 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/a/f325778e59ef490d9532794d8e538020 2024-11-13T07:33:56,583 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/b/616f9a295abc4e1da887326b4c1b976d is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731483236365/Put/seqid=0 2024-11-13T07:33:56,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741912_1090 (size=5524) 2024-11-13T07:33:56,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741912_1090 (size=5524) 2024-11-13T07:33:56,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741912_1090 (size=5524) 2024-11-13T07:33:56,589 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/b/616f9a295abc4e1da887326b4c1b976d 2024-11-13T07:33:56,608 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/c/610bb901575a469e826e1a8a17aef06d is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731483236367/Put/seqid=0 2024-11-13T07:33:56,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741913_1091 (size=5457) 2024-11-13T07:33:56,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741913_1091 (size=5457) 2024-11-13T07:33:56,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741913_1091 (size=5457) 2024-11-13T07:33:56,615 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/c/610bb901575a469e826e1a8a17aef06d 2024-11-13T07:33:56,620 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/a/f325778e59ef490d9532794d8e538020 as hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/a/f325778e59ef490d9532794d8e538020 2024-11-13T07:33:56,625 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/a/f325778e59ef490d9532794d8e538020, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T07:33:56,626 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/b/616f9a295abc4e1da887326b4c1b976d as hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/b/616f9a295abc4e1da887326b4c1b976d 2024-11-13T07:33:56,631 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/b/616f9a295abc4e1da887326b4c1b976d, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T07:33:56,632 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/.tmp/c/610bb901575a469e826e1a8a17aef06d as hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/c/610bb901575a469e826e1a8a17aef06d 2024-11-13T07:33:56,637 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/c/610bb901575a469e826e1a8a17aef06d, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T07:33:56,637 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 8e5384754a2b7eb8e38b90c2b437394b in 92ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T07:33:56,638 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/0000000000000000026 2024-11-13T07:33:56,639 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,639 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,640 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:56,641 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,643 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsAfterAbortingFlush/8e5384754a2b7eb8e38b90c2b437394b/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T07:33:56,643 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e5384754a2b7eb8e38b90c2b437394b; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68169974, jitterRate=0.015811771154403687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:56,644 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e5384754a2b7eb8e38b90c2b437394b 2024-11-13T07:33:56,644 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e5384754a2b7eb8e38b90c2b437394b: Running coprocessor pre-open hook at 1731483236510Writing region info on filesystem at 1731483236510Initializing all the Stores at 1731483236511 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483236511Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483236511Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483236511Obtaining lock to block concurrent updates at 1731483236545 (+34 ms)Preparing flush snapshotting stores in 8e5384754a2b7eb8e38b90c2b437394b at 1731483236545Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731483236545Flushing stores of testReplayEditsAfterAbortingFlush,,1731483227139.8e5384754a2b7eb8e38b90c2b437394b. at 1731483236545Flushing 8e5384754a2b7eb8e38b90c2b437394b/a: creating writer at 1731483236545Flushing 8e5384754a2b7eb8e38b90c2b437394b/a: appending metadata at 1731483236559 (+14 ms)Flushing 8e5384754a2b7eb8e38b90c2b437394b/a: closing flushed file at 1731483236559Flushing 8e5384754a2b7eb8e38b90c2b437394b/b: creating writer at 1731483236570 (+11 ms)Flushing 8e5384754a2b7eb8e38b90c2b437394b/b: appending metadata at 1731483236583 (+13 ms)Flushing 8e5384754a2b7eb8e38b90c2b437394b/b: closing flushed file at 1731483236583Flushing 8e5384754a2b7eb8e38b90c2b437394b/c: creating writer at 1731483236594 (+11 ms)Flushing 8e5384754a2b7eb8e38b90c2b437394b/c: appending metadata at 1731483236607 (+13 ms)Flushing 8e5384754a2b7eb8e38b90c2b437394b/c: closing flushed file at 1731483236607Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a4cd24: reopening flushed file at 1731483236619 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aadd2c2: reopening flushed file at 1731483236625 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17db5ba9: reopening flushed file at 1731483236631 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 8e5384754a2b7eb8e38b90c2b437394b in 92ms, sequenceid=26, compaction requested=false; wal=null at 1731483236637 (+6 ms)Cleaning up temporary data from old regions at 1731483236639 (+2 ms)Running coprocessor post-open hooks at 1731483236644 (+5 ms)Region opened successfully at 1731483236644 2024-11-13T07:33:56,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418 (was 418), OpenFileDescriptor=1182 (was 1118) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=270 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=8390 (was 8411) 2024-11-13T07:33:56,662 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-13T07:33:56,674 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=270, ProcessCount=11, AvailableMemoryMB=8389 2024-11-13T07:33:56,674 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-13T07:33:56,687 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:56,689 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:56,689 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:56,692 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64044839, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-64044839, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:56,703 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64044839/hregion-64044839.1731483236692, exclude list is [], retry=0 2024-11-13T07:33:56,706 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:56,706 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:56,706 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:56,730 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64044839/hregion-64044839.1731483236692 2024-11-13T07:33:56,730 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:56,730 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b9e5518b4d5fe24d55b40eb9bc25ca76, NAME => 'testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:56,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741915_1093 (size=61) 2024-11-13T07:33:56,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741915_1093 (size=61) 2024-11-13T07:33:56,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741915_1093 (size=61) 2024-11-13T07:33:56,740 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:56,742 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,744 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9e5518b4d5fe24d55b40eb9bc25ca76 columnFamilyName a 2024-11-13T07:33:56,744 DEBUG [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:56,744 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(327): Store=b9e5518b4d5fe24d55b40eb9bc25ca76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:56,744 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,746 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,746 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,747 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,749 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:56,749 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9e5518b4d5fe24d55b40eb9bc25ca76; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73384058, jitterRate=0.09350767731666565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:56,750 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9e5518b4d5fe24d55b40eb9bc25ca76: Writing region info on filesystem at 1731483236740Initializing all the Stores at 1731483236741 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483236741Cleaning up temporary data from old regions at 1731483236746 (+5 ms)Region opened successfully at 1731483236750 (+4 ms) 2024-11-13T07:33:56,750 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b9e5518b4d5fe24d55b40eb9bc25ca76, disabling compactions & flushes 2024-11-13T07:33:56,750 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,750 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,750 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. after waiting 0 ms 2024-11-13T07:33:56,750 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,750 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,750 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b9e5518b4d5fe24d55b40eb9bc25ca76: Waiting for close lock at 1731483236750Disabling compacts and flushes for region at 1731483236750Disabling writes for close at 1731483236750Writing region close event to WAL at 1731483236750Closed at 1731483236750 2024-11-13T07:33:56,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741914_1092 (size=175) 2024-11-13T07:33:56,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741914_1092 (size=175) 2024-11-13T07:33:56,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741914_1092 (size=175) 2024-11-13T07:33:56,755 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:56,755 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64044839:(num 1731483236692) 2024-11-13T07:33:56,755 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:56,756 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:56,768 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757, exclude list is [], retry=0 2024-11-13T07:33:56,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:56,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:56,771 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:56,793 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 2024-11-13T07:33:56,793 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:33:56,793 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9e5518b4d5fe24d55b40eb9bc25ca76, NAME => 'testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:56,793 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:56,794 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,794 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,795 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,796 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9e5518b4d5fe24d55b40eb9bc25ca76 columnFamilyName a 2024-11-13T07:33:56,796 DEBUG [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:56,796 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(327): Store=b9e5518b4d5fe24d55b40eb9bc25ca76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:56,796 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,797 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,798 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,798 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,798 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,800 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:56,801 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9e5518b4d5fe24d55b40eb9bc25ca76; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60426826, jitterRate=-0.0995701253414154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:56,801 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9e5518b4d5fe24d55b40eb9bc25ca76: Writing region info on filesystem at 1731483236794Initializing all the Stores at 1731483236795 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483236795Cleaning up temporary data from old regions at 1731483236798 (+3 ms)Region opened successfully at 1731483236801 (+3 ms) 2024-11-13T07:33:56,808 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b9e5518b4d5fe24d55b40eb9bc25ca76, disabling compactions & flushes 2024-11-13T07:33:56,809 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. after waiting 0 ms 2024-11-13T07:33:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,809 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,809 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. 2024-11-13T07:33:56,809 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b9e5518b4d5fe24d55b40eb9bc25ca76: Waiting for close lock at 1731483236808Disabling compacts and flushes for region at 1731483236808Disabling writes for close at 1731483236809 (+1 ms)Writing region close event to WAL at 1731483236809Closed at 1731483236809 2024-11-13T07:33:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741916_1094 (size=1795) 2024-11-13T07:33:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741916_1094 (size=1795) 2024-11-13T07:33:56,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741916_1094 (size=1795) 2024-11-13T07:33:56,826 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757, size=1.8 K (1795bytes) 2024-11-13T07:33:56,826 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 2024-11-13T07:33:56,827 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 after 1ms 2024-11-13T07:33:56,851 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:56,851 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 took 25ms 2024-11-13T07:33:56,853 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 so closing down 2024-11-13T07:33:56,853 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:33:56,854 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731483236757.temp 2024-11-13T07:33:56,877 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000003-wal.1731483236757.temp 2024-11-13T07:33:56,877 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:33:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741917_1095 (size=1795) 2024-11-13T07:33:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741917_1095 (size=1795) 2024-11-13T07:33:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741917_1095 (size=1795) 2024-11-13T07:33:56,884 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000003-wal.1731483236757.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T07:33:56,885 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000003-wal.1731483236757.temp to hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000012 2024-11-13T07:33:56,885 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-13T07:33:56,885 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757, journal: Splitting hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757, size=1.8 K (1795bytes) at 1731483236826Finishing writing output for hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 so closing down at 1731483236853 (+27 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000003-wal.1731483236757.temp at 1731483236877 (+24 ms)3 split writer threads finished at 1731483236877Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000003-wal.1731483236757.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731483236884 (+7 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000003-wal.1731483236757.temp to hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000012 at 1731483236885 (+1 ms)Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731483236885 2024-11-13T07:33:56,887 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483236757 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483236757 2024-11-13T07:33:56,887 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000012 2024-11-13T07:33:56,914 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:57,241 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:57,243 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:57,254 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483237243, exclude list is [], retry=0 2024-11-13T07:33:57,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:57,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:57,257 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:57,281 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483237243 2024-11-13T07:33:57,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:57,281 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9e5518b4d5fe24d55b40eb9bc25ca76, NAME => 'testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:57,281 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:57,282 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,282 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,285 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,286 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9e5518b4d5fe24d55b40eb9bc25ca76 columnFamilyName a 2024-11-13T07:33:57,286 DEBUG [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,287 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(327): Store=b9e5518b4d5fe24d55b40eb9bc25ca76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,287 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,289 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,290 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000012 2024-11-13T07:33:57,315 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:33:57,316 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000012 2024-11-13T07:33:57,316 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b9e5518b4d5fe24d55b40eb9bc25ca76 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T07:33:57,332 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/.tmp/a/555074effb89456caae30829dc1cc296 is 79, key is testDatalossWhenInputError/a:x0/1731483236801/Put/seqid=0 2024-11-13T07:33:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741919_1097 (size=5808) 2024-11-13T07:33:57,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741919_1097 (size=5808) 2024-11-13T07:33:57,338 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/.tmp/a/555074effb89456caae30829dc1cc296 2024-11-13T07:33:57,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741919_1097 (size=5808) 2024-11-13T07:33:57,347 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/.tmp/a/555074effb89456caae30829dc1cc296 as hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/a/555074effb89456caae30829dc1cc296 2024-11-13T07:33:57,355 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/a/555074effb89456caae30829dc1cc296, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T07:33:57,355 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b9e5518b4d5fe24d55b40eb9bc25ca76 in 39ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T07:33:57,356 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/0000000000000000012 2024-11-13T07:33:57,357 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,357 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,360 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,362 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T07:33:57,362 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9e5518b4d5fe24d55b40eb9bc25ca76; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73844891, jitterRate=0.10037462413311005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:57,363 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9e5518b4d5fe24d55b40eb9bc25ca76: Writing region info on filesystem at 1731483237282Initializing all the Stores at 1731483237284 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237284Obtaining lock to block concurrent updates at 1731483237316 (+32 ms)Preparing flush snapshotting stores in b9e5518b4d5fe24d55b40eb9bc25ca76 at 1731483237316Finished memstore snapshotting testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731483237316Flushing stores of testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76. at 1731483237316Flushing b9e5518b4d5fe24d55b40eb9bc25ca76/a: creating writer at 1731483237316Flushing b9e5518b4d5fe24d55b40eb9bc25ca76/a: appending metadata at 1731483237331 (+15 ms)Flushing b9e5518b4d5fe24d55b40eb9bc25ca76/a: closing flushed file at 1731483237331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f70d847: reopening flushed file at 1731483237345 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b9e5518b4d5fe24d55b40eb9bc25ca76 in 39ms, sequenceid=12, compaction requested=false; wal=null at 1731483237355 (+10 ms)Cleaning up temporary data from old regions at 1731483237357 (+2 ms)Region opened successfully at 1731483237363 (+6 ms) 2024-11-13T07:33:57,365 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9e5518b4d5fe24d55b40eb9bc25ca76, NAME => 'testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731483236688.b9e5518b4d5fe24d55b40eb9bc25ca76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,367 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,367 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9e5518b4d5fe24d55b40eb9bc25ca76 columnFamilyName a 2024-11-13T07:33:57,367 DEBUG [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,372 DEBUG [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/a/555074effb89456caae30829dc1cc296 2024-11-13T07:33:57,372 INFO [StoreOpener-b9e5518b4d5fe24d55b40eb9bc25ca76-1 {}] regionserver.HStore(327): Store=b9e5518b4d5fe24d55b40eb9bc25ca76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,372 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,372 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,374 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,374 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,375 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9e5518b4d5fe24d55b40eb9bc25ca76 2024-11-13T07:33:57,377 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testDatalossWhenInputError/b9e5518b4d5fe24d55b40eb9bc25ca76/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T07:33:57,378 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9e5518b4d5fe24d55b40eb9bc25ca76; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67471249, jitterRate=0.005399957299232483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:33:57,379 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9e5518b4d5fe24d55b40eb9bc25ca76: Writing region info on filesystem at 1731483237366Initializing all the Stores at 1731483237366Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237366Cleaning up temporary data from old regions at 1731483237374 (+8 ms)Region opened successfully at 1731483237378 (+4 ms) 2024-11-13T07:33:57,397 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:48372 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:34648 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:37476 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:34758 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:48422 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-951039070-172.17.0.3-1731483201138:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:37546 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=270 (was 270), ProcessCount=11 (was 11), AvailableMemoryMB=8365 (was 8389) 2024-11-13T07:33:57,397 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T07:33:57,410 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=270, ProcessCount=11, AvailableMemoryMB=8365 2024-11-13T07:33:57,411 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T07:33:57,427 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:57,429 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:33:57,429 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:33:57,431 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33459796, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-33459796, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:57,442 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33459796/hregion-33459796.1731483237432, exclude list is [], retry=0 2024-11-13T07:33:57,445 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:57,445 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:57,446 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:57,478 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33459796/hregion-33459796.1731483237432 2024-11-13T07:33:57,479 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:33:57,479 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e2980eca3f4eb49d98c25496721d0d99, NAME => 'testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:33:57,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741921_1099 (size=63) 2024-11-13T07:33:57,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741921_1099 (size=63) 2024-11-13T07:33:57,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741921_1099 (size=63) 2024-11-13T07:33:57,489 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:57,490 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,491 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName a 2024-11-13T07:33:57,492 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,492 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,492 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,493 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName b 2024-11-13T07:33:57,493 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,494 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,494 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,495 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName c 2024-11-13T07:33:57,495 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,495 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,495 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,497 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,497 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,497 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:57,498 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,501 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:33:57,501 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2980eca3f4eb49d98c25496721d0d99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60652206, jitterRate=-0.09621170163154602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:57,501 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2980eca3f4eb49d98c25496721d0d99: Writing region info on filesystem at 1731483237489Initializing all the Stores at 1731483237490 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237490Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237490Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237490Cleaning up temporary data from old regions at 1731483237497 (+7 ms)Region opened successfully at 1731483237501 (+4 ms) 2024-11-13T07:33:57,502 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2980eca3f4eb49d98c25496721d0d99, disabling compactions & flushes 2024-11-13T07:33:57,502 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:33:57,502 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:33:57,502 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. after waiting 0 ms 2024-11-13T07:33:57,502 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:33:57,502 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:33:57,502 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2980eca3f4eb49d98c25496721d0d99: Waiting for close lock at 1731483237501Disabling compacts and flushes for region at 1731483237501Disabling writes for close at 1731483237502 (+1 ms)Writing region close event to WAL at 1731483237502Closed at 1731483237502 2024-11-13T07:33:57,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741920_1098 (size=175) 2024-11-13T07:33:57,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741920_1098 (size=175) 2024-11-13T07:33:57,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741920_1098 (size=175) 2024-11-13T07:33:57,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:33:57,508 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33459796:(num 1731483237432) 2024-11-13T07:33:57,508 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:33:57,509 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:33:57,523 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510, exclude list is [], retry=0 2024-11-13T07:33:57,525 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:33:57,525 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:33:57,526 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:33:57,549 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 2024-11-13T07:33:57,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:33:57,549 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e2980eca3f4eb49d98c25496721d0d99, NAME => 'testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:33:57,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:33:57,550 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,550 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,551 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,552 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName a 2024-11-13T07:33:57,552 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,552 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,552 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,553 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName b 2024-11-13T07:33:57,553 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,553 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,553 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,554 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName c 2024-11-13T07:33:57,554 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:33:57,554 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:33:57,555 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,557 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,557 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,558 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:33:57,559 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:33:57,560 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2980eca3f4eb49d98c25496721d0d99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72618521, jitterRate=0.0821002870798111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:33:57,560 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2980eca3f4eb49d98c25496721d0d99: Writing region info on filesystem at 1731483237550Initializing all the Stores at 1731483237551 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237551Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237551Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483237551Cleaning up temporary data from old regions at 1731483237557 (+6 ms)Region opened successfully at 1731483237560 (+3 ms) 2024-11-13T07:33:57,564 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731483237564/Put/seqid=0 2024-11-13T07:33:57,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741923_1101 (size=4875) 2024-11-13T07:33:57,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741923_1101 (size=4875) 2024-11-13T07:33:57,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741923_1101 (size=4875) 2024-11-13T07:33:57,575 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731483237575/Put/seqid=0 2024-11-13T07:33:57,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741924_1102 (size=4875) 2024-11-13T07:33:57,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741924_1102 (size=4875) 2024-11-13T07:33:57,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741924_1102 (size=4875) 2024-11-13T07:33:57,586 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731483237586/Put/seqid=0 2024-11-13T07:33:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741925_1103 (size=4875) 2024-11-13T07:33:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741925_1103 (size=4875) 2024-11-13T07:33:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741925_1103 (size=4875) 2024-11-13T07:33:57,592 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in e2980eca3f4eb49d98c25496721d0d99/a 2024-11-13T07:33:57,595 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T07:33:57,595 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T07:33:57,596 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in e2980eca3f4eb49d98c25496721d0d99/a 2024-11-13T07:33:57,599 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T07:33:57,599 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T07:33:57,599 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in e2980eca3f4eb49d98c25496721d0d99/a 2024-11-13T07:33:57,602 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T07:33:57,602 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T07:33:57,602 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e2980eca3f4eb49d98c25496721d0d99 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T07:33:57,615 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/.tmp/a/79fb5d00829446a5aebe5a6a05be50a9 is 55, key is testCompactedBulkLoadedFiles/a:a/1731483237561/Put/seqid=0 2024-11-13T07:33:57,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741926_1104 (size=5107) 2024-11-13T07:33:57,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741926_1104 (size=5107) 2024-11-13T07:33:57,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741926_1104 (size=5107) 2024-11-13T07:33:57,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/.tmp/a/79fb5d00829446a5aebe5a6a05be50a9 2024-11-13T07:33:57,633 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/.tmp/a/79fb5d00829446a5aebe5a6a05be50a9 as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9 2024-11-13T07:33:57,637 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T07:33:57,639 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for e2980eca3f4eb49d98c25496721d0d99 in 37ms, sequenceid=4, compaction requested=false 2024-11-13T07:33:57,639 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e2980eca3f4eb49d98c25496721d0d99: 2024-11-13T07:33:57,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ 2024-11-13T07:33:57,641 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ 2024-11-13T07:33:57,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ 2024-11-13T07:33:57,643 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile0 into e2980eca3f4eb49d98c25496721d0d99/a as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ - updating store file list. 2024-11-13T07:33:57,647 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T07:33:57,647 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ into e2980eca3f4eb49d98c25496721d0d99/a 2024-11-13T07:33:57,647 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile0 into e2980eca3f4eb49d98c25496721d0d99/a (new location: hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_) 2024-11-13T07:33:57,649 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile1 into e2980eca3f4eb49d98c25496721d0d99/a as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ - updating store file list. 2024-11-13T07:33:57,653 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 48dad6d180484909a3ec64d259768a38_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T07:33:57,653 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ into e2980eca3f4eb49d98c25496721d0d99/a 2024-11-13T07:33:57,653 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile1 into e2980eca3f4eb49d98c25496721d0d99/a (new location: hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_) 2024-11-13T07:33:57,654 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile2 into e2980eca3f4eb49d98c25496721d0d99/a as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ - updating store file list. 2024-11-13T07:33:57,661 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 60622b7e4f2040d2867e25019422fb0a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T07:33:57,661 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ into e2980eca3f4eb49d98c25496721d0d99/a 2024-11-13T07:33:57,661 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35647/hbase/testCompactedBulkLoadedFiles/hfile2 into e2980eca3f4eb49d98c25496721d0d99/a (new location: hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_) 2024-11-13T07:33:57,670 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T07:33:57,670 DEBUG [Time-limited test {}] regionserver.HStore(1541): e2980eca3f4eb49d98c25496721d0d99/a is initiating major compaction (all files) 2024-11-13T07:33:57,670 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e2980eca3f4eb49d98c25496721d0d99/a in testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:33:57,670 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_] into tmpdir=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/.tmp, totalSize=19.3 K 2024-11-13T07:33:57,671 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 79fb5d00829446a5aebe5a6a05be50a9, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731483237561 2024-11-13T07:33:57,671 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T07:33:57,672 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 48dad6d180484909a3ec64d259768a38_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T07:33:57,672 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 60622b7e4f2040d2867e25019422fb0a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T07:33:57,683 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/.tmp/a/5e86b048f1fb4cd9a248a1a4c129bf3a is 55, key is testCompactedBulkLoadedFiles/a:a/1731483237561/Put/seqid=0 2024-11-13T07:33:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741927_1105 (size=6154) 2024-11-13T07:33:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741927_1105 (size=6154) 2024-11-13T07:33:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741927_1105 (size=6154) 2024-11-13T07:33:57,695 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/.tmp/a/5e86b048f1fb4cd9a248a1a4c129bf3a as hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/5e86b048f1fb4cd9a248a1a4c129bf3a 2024-11-13T07:33:57,701 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in e2980eca3f4eb49d98c25496721d0d99/a of e2980eca3f4eb49d98c25496721d0d99 into 5e86b048f1fb4cd9a248a1a4c129bf3a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T07:33:57,701 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e2980eca3f4eb49d98c25496721d0d99: 2024-11-13T07:33:57,701 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T07:33:57,701 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T07:33:57,730 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510, size=0 (0bytes) 2024-11-13T07:33:57,730 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 might be still open, length is 0 2024-11-13T07:33:57,730 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 2024-11-13T07:33:57,730 WARN [IPC Server handler 4 on default port 35647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-13T07:33:57,731 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 after 0ms 2024-11-13T07:34:00,570 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:34780 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:35659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34780 dst: /127.0.0.1:35659 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35659 remote=/127.0.0.1:34780]. Total timeout mills is 60000, 57131 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:00,571 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:37588 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37588 dst: /127.0.0.1:41959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:00,571 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:48464 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48464 dst: /127.0.0.1:38619 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:00,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741922_1106 (size=1794) 2024-11-13T07:34:00,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741922_1106 (size=1794) 2024-11-13T07:34:00,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741922_1106 (size=1794) 2024-11-13T07:34:01,732 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 after 4001ms 2024-11-13T07:34:01,797 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:34:01,797 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 took 4067ms 2024-11-13T07:34:01,799 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510; continuing. 2024-11-13T07:34:01,799 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 so closing down 2024-11-13T07:34:01,799 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:34:01,800 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731483237510.temp 2024-11-13T07:34:01,826 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000003-wal.1731483237510.temp 2024-11-13T07:34:01,826 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:34:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741928_1107 (size=800) 2024-11-13T07:34:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741928_1107 (size=800) 2024-11-13T07:34:01,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741928_1107 (size=800) 2024-11-13T07:34:01,833 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000003-wal.1731483237510.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T07:34:01,834 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000003-wal.1731483237510.temp to hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000008 2024-11-13T07:34:01,834 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T07:34:01,834 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510, journal: Splitting hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510, size=0 (0bytes) at 1731483237730Finishing writing output for hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 so closing down at 1731483241799 (+4069 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000003-wal.1731483237510.temp at 1731483241826 (+27 ms)3 split writer threads finished at 1731483241826Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000003-wal.1731483237510.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731483241833 (+7 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000003-wal.1731483237510.temp to hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000008 at 1731483241834 (+1 ms)Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510, size=0, length=0, corrupted=false, cancelled=false at 1731483241834 2024-11-13T07:34:01,835 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483237510 2024-11-13T07:34:01,836 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000008 2024-11-13T07:34:01,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:34:01,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:34:01,854 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483241839, exclude list is [], retry=0 2024-11-13T07:34:01,857 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:34:01,857 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:34:01,857 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:34:01,881 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483241839 2024-11-13T07:34:01,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:34:01,881 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e2980eca3f4eb49d98c25496721d0d99, NAME => 'testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:34:01,882 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:34:01,882 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,882 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,883 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,884 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName a 2024-11-13T07:34:01,884 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:01,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:34:01,890 DEBUG [StoreFileOpener-e2980eca3f4eb49d98c25496721d0d99-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 48dad6d180484909a3ec64d259768a38_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T07:34:01,890 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ 2024-11-13T07:34:01,894 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/5e86b048f1fb4cd9a248a1a4c129bf3a 2024-11-13T07:34:01,898 DEBUG [StoreFileOpener-e2980eca3f4eb49d98c25496721d0d99-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 60622b7e4f2040d2867e25019422fb0a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T07:34:01,899 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ 2024-11-13T07:34:01,903 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9 2024-11-13T07:34:01,907 DEBUG [StoreFileOpener-e2980eca3f4eb49d98c25496721d0d99-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T07:34:01,907 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ 2024-11-13T07:34:01,908 WARN [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4fbedb5e 2024-11-13T07:34:01,908 WARN [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4fbedb5e 2024-11-13T07:34:01,908 WARN [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4fbedb5e 2024-11-13T07:34:01,908 WARN [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4fbedb5e 2024-11-13T07:34:01,908 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_] to archive 2024-11-13T07:34:01,909 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T07:34:01,910 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ to hdfs://localhost:35647/hbase/archive/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/48dad6d180484909a3ec64d259768a38_SeqId_4_ 2024-11-13T07:34:01,912 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ to hdfs://localhost:35647/hbase/archive/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/60622b7e4f2040d2867e25019422fb0a_SeqId_4_ 2024-11-13T07:34:01,913 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9 to hdfs://localhost:35647/hbase/archive/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/79fb5d00829446a5aebe5a6a05be50a9 2024-11-13T07:34:01,914 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ to hdfs://localhost:35647/hbase/archive/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/a/edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_ 2024-11-13T07:34:01,914 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:01,914 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,915 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName b 2024-11-13T07:34:01,915 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:01,916 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:01,916 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,916 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2980eca3f4eb49d98c25496721d0d99 columnFamilyName c 2024-11-13T07:34:01,917 DEBUG [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:01,917 INFO [StoreOpener-e2980eca3f4eb49d98c25496721d0d99-1 {}] regionserver.HStore(327): Store=e2980eca3f4eb49d98c25496721d0d99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:01,917 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,918 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,919 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,920 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000008 2024-11-13T07:34:01,949 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:34:01,952 DEBUG [Time-limited test {}] regionserver.HRegion(5836): e2980eca3f4eb49d98c25496721d0d99 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "e2980eca3f4eb49d98c25496721d0d99" family_name: "a" compaction_input: "79fb5d00829446a5aebe5a6a05be50a9" compaction_input: "edd7e3bc00e7466dabc28eac6da842f0_SeqId_4_" compaction_input: "48dad6d180484909a3ec64d259768a38_SeqId_4_" compaction_input: "60622b7e4f2040d2867e25019422fb0a_SeqId_4_" compaction_output: "5e86b048f1fb4cd9a248a1a4c129bf3a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T07:34:01,952 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T07:34:01,952 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000008 2024-11-13T07:34:01,953 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/0000000000000000008 2024-11-13T07:34:01,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,955 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:34:01,956 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2980eca3f4eb49d98c25496721d0d99 2024-11-13T07:34:01,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testCompactedBulkLoadedFiles/e2980eca3f4eb49d98c25496721d0d99/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:34:01,960 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2980eca3f4eb49d98c25496721d0d99; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72014875, jitterRate=0.07310526072978973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:34:01,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2980eca3f4eb49d98c25496721d0d99: Writing region info on filesystem at 1731483241882Initializing all the Stores at 1731483241883 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483241883Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483241883Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483241883Cleaning up temporary data from old regions at 1731483241954 (+71 ms)Region opened successfully at 1731483241960 (+6 ms) 2024-11-13T07:34:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2980eca3f4eb49d98c25496721d0d99, disabling compactions & flushes 2024-11-13T07:34:01,962 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:34:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:34:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. after waiting 0 ms 2024-11-13T07:34:01,962 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:34:01,963 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731483237428.e2980eca3f4eb49d98c25496721d0d99. 2024-11-13T07:34:01,963 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2980eca3f4eb49d98c25496721d0d99: Waiting for close lock at 1731483241962Disabling compacts and flushes for region at 1731483241962Disabling writes for close at 1731483241962Writing region close event to WAL at 1731483241963 (+1 ms)Closed at 1731483241963 2024-11-13T07:34:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741929_1108 (size=175) 2024-11-13T07:34:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741929_1108 (size=175) 2024-11-13T07:34:01,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741929_1108 (size=175) 2024-11-13T07:34:01,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:34:01,968 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731483241839) 2024-11-13T07:34:01,981 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=442 (was 428) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1307153954_22 at /127.0.0.1:51786 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1307153954_22 at /127.0.0.1:48924 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1307153954_22 at /127.0.0.1:37628 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35647 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=248 (was 270), ProcessCount=11 (was 11), AvailableMemoryMB=8357 (was 8365) 2024-11-13T07:34:01,981 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T07:34:01,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=8357 2024-11-13T07:34:01,994 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T07:34:02,006 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:34:02,008 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:34:02,009 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:34:02,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50334485, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/hregion-50334485, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:34:02,023 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50334485/hregion-50334485.1731483242011, exclude list is [], retry=0 2024-11-13T07:34:02,026 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:34:02,026 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:34:02,026 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:34:02,051 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50334485/hregion-50334485.1731483242011 2024-11-13T07:34:02,051 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:34:02,052 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 41d5462e437229fc03bf5369807dbf31, NAME => 'testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35647/hbase 2024-11-13T07:34:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741931_1110 (size=67) 2024-11-13T07:34:02,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741931_1110 (size=67) 2024-11-13T07:34:02,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741931_1110 (size=67) 2024-11-13T07:34:02,059 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:34:02,061 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,062 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName a 2024-11-13T07:34:02,062 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,063 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,063 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,065 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName b 2024-11-13T07:34:02,065 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,065 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,065 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,067 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName c 2024-11-13T07:34:02,067 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,067 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,067 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,068 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,069 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,070 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,070 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,070 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:34:02,072 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,074 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:34:02,074 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41d5462e437229fc03bf5369807dbf31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68243015, jitterRate=0.016900166869163513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:34:02,075 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41d5462e437229fc03bf5369807dbf31: Writing region info on filesystem at 1731483242059Initializing all the Stores at 1731483242060 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242060Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242061 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242061Cleaning up temporary data from old regions at 1731483242070 (+9 ms)Region opened successfully at 1731483242075 (+5 ms) 2024-11-13T07:34:02,075 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41d5462e437229fc03bf5369807dbf31, disabling compactions & flushes 2024-11-13T07:34:02,075 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,075 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,075 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. after waiting 0 ms 2024-11-13T07:34:02,075 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,076 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,076 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41d5462e437229fc03bf5369807dbf31: Waiting for close lock at 1731483242075Disabling compacts and flushes for region at 1731483242075Disabling writes for close at 1731483242075Writing region close event to WAL at 1731483242075Closed at 1731483242075 2024-11-13T07:34:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741930_1109 (size=175) 2024-11-13T07:34:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741930_1109 (size=175) 2024-11-13T07:34:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741930_1109 (size=175) 2024-11-13T07:34:02,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:34:02,080 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50334485:(num 1731483242011) 2024-11-13T07:34:02,080 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:34:02,082 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:34:02,097 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082, exclude list is [], retry=0 2024-11-13T07:34:02,100 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:34:02,100 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:34:02,100 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:34:02,124 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 2024-11-13T07:34:02,124 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:41899:41899)] 2024-11-13T07:34:02,124 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41d5462e437229fc03bf5369807dbf31, NAME => 'testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:34:02,124 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:34:02,125 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,125 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,126 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,127 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName a 2024-11-13T07:34:02,127 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,127 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,128 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,128 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName b 2024-11-13T07:34:02,128 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,129 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,129 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,130 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName c 2024-11-13T07:34:02,130 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,130 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,130 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,131 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,132 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,133 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,133 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,133 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:34:02,134 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,135 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41d5462e437229fc03bf5369807dbf31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73353936, jitterRate=0.09305882453918457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:34:02,135 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41d5462e437229fc03bf5369807dbf31: Writing region info on filesystem at 1731483242125Initializing all the Stores at 1731483242126 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242126Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242126Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242126Cleaning up temporary data from old regions at 1731483242133 (+7 ms)Region opened successfully at 1731483242135 (+2 ms) 2024-11-13T07:34:02,142 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41d5462e437229fc03bf5369807dbf31 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T07:34:02,156 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/a/fe4191b0f4184a3ba6290d762998c116 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731483242135/Put/seqid=0 2024-11-13T07:34:02,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741933_1112 (size=5958) 2024-11-13T07:34:02,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741933_1112 (size=5958) 2024-11-13T07:34:02,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741933_1112 (size=5958) 2024-11-13T07:34:02,162 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/a/fe4191b0f4184a3ba6290d762998c116 2024-11-13T07:34:02,167 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/a/fe4191b0f4184a3ba6290d762998c116 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/a/fe4191b0f4184a3ba6290d762998c116 2024-11-13T07:34:02,171 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/a/fe4191b0f4184a3ba6290d762998c116, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T07:34:02,172 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 41d5462e437229fc03bf5369807dbf31 in 30ms, sequenceid=13, compaction requested=false 2024-11-13T07:34:02,172 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 41d5462e437229fc03bf5369807dbf31: 2024-11-13T07:34:02,188 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41d5462e437229fc03bf5369807dbf31, disabling compactions & flushes 2024-11-13T07:34:02,188 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,188 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,188 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. after waiting 0 ms 2024-11-13T07:34:02,188 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,188 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,189 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:02,189 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41d5462e437229fc03bf5369807dbf31: Waiting for close lock at 1731483242188Disabling compacts and flushes for region at 1731483242188Disabling writes for close at 1731483242188Writing region close event to WAL at 1731483242188Closed at 1731483242188 2024-11-13T07:34:02,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741932_1111 (size=6243) 2024-11-13T07:34:02,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741932_1111 (size=6243) 2024-11-13T07:34:02,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741932_1111 (size=6243) 2024-11-13T07:34:02,205 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082, size=6.1 K (6243bytes) 2024-11-13T07:34:02,205 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 2024-11-13T07:34:02,205 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 after 0ms 2024-11-13T07:34:02,229 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:34:02,230 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 took 25ms 2024-11-13T07:34:02,231 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 so closing down 2024-11-13T07:34:02,231 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:34:02,233 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731483242082.temp 2024-11-13T07:34:02,257 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000003-wal.1731483242082.temp 2024-11-13T07:34:02,257 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:34:02,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741934_1113 (size=5575) 2024-11-13T07:34:02,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741934_1113 (size=5575) 2024-11-13T07:34:02,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741934_1113 (size=5575) 2024-11-13T07:34:02,264 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000003-wal.1731483242082.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T07:34:02,265 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000003-wal.1731483242082.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000035 2024-11-13T07:34:02,266 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-13T07:34:02,266 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082, journal: Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082, size=6.1 K (6243bytes) at 1731483242205Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 so closing down at 1731483242231 (+26 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000003-wal.1731483242082.temp at 1731483242257 (+26 ms)3 split writer threads finished at 1731483242257Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000003-wal.1731483242082.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731483242264 (+7 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000003-wal.1731483242082.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000035 at 1731483242266 (+2 ms)Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731483242266 2024-11-13T07:34:02,267 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242082 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483242082 2024-11-13T07:34:02,268 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000035 2024-11-13T07:34:02,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:34:02,269 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:34:02,283 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270, exclude list is [], retry=0 2024-11-13T07:34:02,286 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:34:02,286 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:34:02,286 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:34:02,310 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 2024-11-13T07:34:02,311 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34891:34891),(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579)] 2024-11-13T07:34:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41d5462e437229fc03bf5369807dbf31, NAME => 'testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:34:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:34:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,311 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,312 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,313 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName a 2024-11-13T07:34:02,313 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,318 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/a/fe4191b0f4184a3ba6290d762998c116 2024-11-13T07:34:02,318 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,319 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,319 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName b 2024-11-13T07:34:02,319 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,320 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,320 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,320 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName c 2024-11-13T07:34:02,320 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:02,321 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:02,321 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,323 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,323 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000035 2024-11-13T07:34:02,348 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:34:02,349 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000035 2024-11-13T07:34:02,350 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41d5462e437229fc03bf5369807dbf31 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T07:34:02,363 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/b/adb651b92334460094b9ef8b13bf39b8 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731483242172/Put/seqid=0 2024-11-13T07:34:02,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741936_1115 (size=5958) 2024-11-13T07:34:02,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741936_1115 (size=5958) 2024-11-13T07:34:02,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741936_1115 (size=5958) 2024-11-13T07:34:02,369 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/b/adb651b92334460094b9ef8b13bf39b8 2024-11-13T07:34:02,386 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/c/b3ec7ee554b54f069727823ea3ca0057 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731483242179/Put/seqid=0 2024-11-13T07:34:02,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741937_1116 (size=5958) 2024-11-13T07:34:02,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741937_1116 (size=5958) 2024-11-13T07:34:02,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741937_1116 (size=5958) 2024-11-13T07:34:02,393 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/c/b3ec7ee554b54f069727823ea3ca0057 2024-11-13T07:34:02,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/b/adb651b92334460094b9ef8b13bf39b8 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/b/adb651b92334460094b9ef8b13bf39b8 2024-11-13T07:34:02,402 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/b/adb651b92334460094b9ef8b13bf39b8, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T07:34:02,403 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/c/b3ec7ee554b54f069727823ea3ca0057 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/c/b3ec7ee554b54f069727823ea3ca0057 2024-11-13T07:34:02,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/c/b3ec7ee554b54f069727823ea3ca0057, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T07:34:02,408 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 41d5462e437229fc03bf5369807dbf31 in 59ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T07:34:02,408 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000035 2024-11-13T07:34:02,409 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,409 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,410 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:34:02,411 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:02,413 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T07:34:02,414 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41d5462e437229fc03bf5369807dbf31; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68051341, jitterRate=0.014044001698493958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:34:02,414 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41d5462e437229fc03bf5369807dbf31: Writing region info on filesystem at 1731483242311Initializing all the Stores at 1731483242312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242312Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242312Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483242312Obtaining lock to block concurrent updates at 1731483242350 (+38 ms)Preparing flush snapshotting stores in 41d5462e437229fc03bf5369807dbf31 at 1731483242350Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731483242350Flushing stores of testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. at 1731483242350Flushing 41d5462e437229fc03bf5369807dbf31/b: creating writer at 1731483242350Flushing 41d5462e437229fc03bf5369807dbf31/b: appending metadata at 1731483242362 (+12 ms)Flushing 41d5462e437229fc03bf5369807dbf31/b: closing flushed file at 1731483242362Flushing 41d5462e437229fc03bf5369807dbf31/c: creating writer at 1731483242373 (+11 ms)Flushing 41d5462e437229fc03bf5369807dbf31/c: appending metadata at 1731483242386 (+13 ms)Flushing 41d5462e437229fc03bf5369807dbf31/c: closing flushed file at 1731483242386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52b7e483: reopening flushed file at 1731483242397 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@367f2fbb: reopening flushed file at 1731483242402 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 41d5462e437229fc03bf5369807dbf31 in 59ms, sequenceid=35, compaction requested=false; wal=null at 1731483242408 (+6 ms)Cleaning up temporary data from old regions at 1731483242409 (+1 ms)Region opened successfully at 1731483242414 (+5 ms) 2024-11-13T07:34:02,472 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270, size=0 (0bytes) 2024-11-13T07:34:02,473 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 might be still open, length is 0 2024-11-13T07:34:02,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 2024-11-13T07:34:02,473 WARN [IPC Server handler 3 on default port 35647 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-13T07:34:02,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 after 0ms 2024-11-13T07:34:03,569 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:48978 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:35659:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48978 dst: /127.0.0.1:35659 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35659 remote=/127.0.0.1:48978]. Total timeout mills is 60000, 58871 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:03,571 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:51864 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51864 dst: /127.0.0.1:38619 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:03,571 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484725256_22 at /127.0.0.1:36676 [Receiving block BP-951039070-172.17.0.3-1731483201138:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36676 dst: /127.0.0.1:41959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:03,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741935_1117 (size=5567) 2024-11-13T07:34:03,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741935_1117 (size=5567) 2024-11-13T07:34:03,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741935_1117 (size=5567) 2024-11-13T07:34:05,857 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T07:34:05,857 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T07:34:05,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T07:34:05,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T07:34:06,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 after 4002ms 2024-11-13T07:34:06,506 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:34:06,506 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 took 4034ms 2024-11-13T07:34:06,508 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270; continuing. 2024-11-13T07:34:06,508 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 so closing down 2024-11-13T07:34:06,508 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:34:06,509 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731483242270.temp 2024-11-13T07:34:06,537 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000037-wal.1731483242270.temp 2024-11-13T07:34:06,537 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:34:06,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741938_1118 (size=5575) 2024-11-13T07:34:06,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741938_1118 (size=5575) 2024-11-13T07:34:06,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741938_1118 (size=5575) 2024-11-13T07:34:06,543 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000037-wal.1731483242270.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T07:34:06,544 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000037-wal.1731483242270.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000066 2024-11-13T07:34:06,545 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T07:34:06,545 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270, journal: Splitting hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270, size=0 (0bytes) at 1731483242473Finishing writing output for hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 so closing down at 1731483246508 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000037-wal.1731483242270.temp at 1731483246537 (+29 ms)3 split writer threads finished at 1731483246537Closed recovered edits writer path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000037-wal.1731483242270.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731483246543 (+6 ms)Rename recovered edits hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000037-wal.1731483242270.temp to hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000066 at 1731483246545 (+2 ms)Processed 30 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270, size=0, length=0, corrupted=false, cancelled=false at 1731483246545 2024-11-13T07:34:06,546 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 to hdfs://localhost:35647/hbase/oldWALs/wal.1731483242270 2024-11-13T07:34:06,547 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000066 2024-11-13T07:34:06,547 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T07:34:06,548 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35647/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006, archiveDir=hdfs://localhost:35647/hbase/oldWALs, maxLogs=32 2024-11-13T07:34:06,562 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483246549, exclude list is [], retry=0 2024-11-13T07:34:06,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38619,DS-dc23322e-2bc3-425b-94e5-86976802cb15,DISK] 2024-11-13T07:34:06,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41959,DS-e7d79592-0a9b-4c62-b9a7-8a744931c38f,DISK] 2024-11-13T07:34:06,564 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35659,DS-e22d148f-6c2d-42ec-9e8d-9ed8cec41862,DISK] 2024-11-13T07:34:06,590 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483246549 2024-11-13T07:34:06,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41899:41899),(127.0.0.1/127.0.0.1:34579:34579),(127.0.0.1/127.0.0.1:34891:34891)] 2024-11-13T07:34:06,590 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:34:06,592 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:06,593 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName a 2024-11-13T07:34:06,593 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:06,598 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/a/fe4191b0f4184a3ba6290d762998c116 2024-11-13T07:34:06,598 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:06,598 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:06,599 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName b 2024-11-13T07:34:06,599 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:06,604 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/b/adb651b92334460094b9ef8b13bf39b8 2024-11-13T07:34:06,604 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:06,604 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:06,605 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41d5462e437229fc03bf5369807dbf31 columnFamilyName c 2024-11-13T07:34:06,605 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:34:06,609 DEBUG [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/c/b3ec7ee554b54f069727823ea3ca0057 2024-11-13T07:34:06,610 INFO [StoreOpener-41d5462e437229fc03bf5369807dbf31-1 {}] regionserver.HStore(327): Store=41d5462e437229fc03bf5369807dbf31/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:34:06,610 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:06,611 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:06,612 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:06,612 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000066 2024-11-13T07:34:06,636 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T07:34:06,639 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000066 2024-11-13T07:34:06,640 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41d5462e437229fc03bf5369807dbf31 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T07:34:06,653 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/a/b665434252f849358a40e05ffbd2fc97 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731483242420/Put/seqid=0 2024-11-13T07:34:06,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741940_1120 (size=5958) 2024-11-13T07:34:06,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741940_1120 (size=5958) 2024-11-13T07:34:06,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741940_1120 (size=5958) 2024-11-13T07:34:07,061 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/a/b665434252f849358a40e05ffbd2fc97 2024-11-13T07:34:07,116 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/b/c0fd416a4fa9472ab639fb30a44dc8f7 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731483242426/Put/seqid=0 2024-11-13T07:34:07,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741941_1121 (size=5958) 2024-11-13T07:34:07,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741941_1121 (size=5958) 2024-11-13T07:34:07,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741941_1121 (size=5958) 2024-11-13T07:34:07,127 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/b/c0fd416a4fa9472ab639fb30a44dc8f7 2024-11-13T07:34:07,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/c/3b21371fa0094ed79a28557ce3247932 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731483242433/Put/seqid=0 2024-11-13T07:34:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741942_1122 (size=5958) 2024-11-13T07:34:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741942_1122 (size=5958) 2024-11-13T07:34:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741942_1122 (size=5958) 2024-11-13T07:34:07,150 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/c/3b21371fa0094ed79a28557ce3247932 2024-11-13T07:34:07,155 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/a/b665434252f849358a40e05ffbd2fc97 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/a/b665434252f849358a40e05ffbd2fc97 2024-11-13T07:34:07,159 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/a/b665434252f849358a40e05ffbd2fc97, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T07:34:07,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/b/c0fd416a4fa9472ab639fb30a44dc8f7 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/b/c0fd416a4fa9472ab639fb30a44dc8f7 2024-11-13T07:34:07,164 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/b/c0fd416a4fa9472ab639fb30a44dc8f7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T07:34:07,165 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/.tmp/c/3b21371fa0094ed79a28557ce3247932 as hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/c/3b21371fa0094ed79a28557ce3247932 2024-11-13T07:34:07,170 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/c/3b21371fa0094ed79a28557ce3247932, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T07:34:07,171 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 41d5462e437229fc03bf5369807dbf31 in 531ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T07:34:07,171 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/0000000000000000066 2024-11-13T07:34:07,172 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:07,173 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:07,173 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T07:34:07,175 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41d5462e437229fc03bf5369807dbf31 2024-11-13T07:34:07,177 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/hbase/data/default/testReplayEditsWrittenViaHRegion/41d5462e437229fc03bf5369807dbf31/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T07:34:07,178 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41d5462e437229fc03bf5369807dbf31; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64200042, jitterRate=-0.04334482550621033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T07:34:07,178 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41d5462e437229fc03bf5369807dbf31: Writing region info on filesystem at 1731483246590Initializing all the Stores at 1731483246591 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483246591Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483246592 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731483246592Obtaining lock to block concurrent updates at 1731483246640 (+48 ms)Preparing flush snapshotting stores in 41d5462e437229fc03bf5369807dbf31 at 1731483246640Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731483246640Flushing stores of testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. at 1731483246640Flushing 41d5462e437229fc03bf5369807dbf31/a: creating writer at 1731483246640Flushing 41d5462e437229fc03bf5369807dbf31/a: appending metadata at 1731483246652 (+12 ms)Flushing 41d5462e437229fc03bf5369807dbf31/a: closing flushed file at 1731483246652Flushing 41d5462e437229fc03bf5369807dbf31/b: creating writer at 1731483247095 (+443 ms)Flushing 41d5462e437229fc03bf5369807dbf31/b: appending metadata at 1731483247116 (+21 ms)Flushing 41d5462e437229fc03bf5369807dbf31/b: closing flushed file at 1731483247116Flushing 41d5462e437229fc03bf5369807dbf31/c: creating writer at 1731483247130 (+14 ms)Flushing 41d5462e437229fc03bf5369807dbf31/c: appending metadata at 1731483247143 (+13 ms)Flushing 41d5462e437229fc03bf5369807dbf31/c: closing flushed file at 1731483247143Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13054e3f: reopening flushed file at 1731483247154 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cac21a3: reopening flushed file at 1731483247159 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32d8d11a: reopening flushed file at 1731483247164 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 41d5462e437229fc03bf5369807dbf31 in 531ms, sequenceid=66, compaction requested=false; wal=null at 1731483247171 (+7 ms)Cleaning up temporary data from old regions at 1731483247173 (+2 ms)Region opened successfully at 1731483247178 (+5 ms) 2024-11-13T07:34:07,189 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41d5462e437229fc03bf5369807dbf31, disabling compactions & flushes 2024-11-13T07:34:07,189 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:07,189 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:07,189 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. after waiting 0 ms 2024-11-13T07:34:07,189 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:07,190 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731483242007.41d5462e437229fc03bf5369807dbf31. 2024-11-13T07:34:07,190 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41d5462e437229fc03bf5369807dbf31: Waiting for close lock at 1731483247189Disabling compacts and flushes for region at 1731483247189Disabling writes for close at 1731483247189Writing region close event to WAL at 1731483247190 (+1 ms)Closed at 1731483247190 2024-11-13T07:34:07,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741939_1119 (size=175) 2024-11-13T07:34:07,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741939_1119 (size=175) 2024-11-13T07:34:07,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741939_1119 (size=175) 2024-11-13T07:34:07,194 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T07:34:07,194 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731483246549) 2024-11-13T07:34:07,206 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=446 (was 442) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-517323758_22 at /127.0.0.1:51882 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-517323758_22 at /127.0.0.1:49048 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-517323758_22 at /127.0.0.1:36726 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35647 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35647 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=244 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=8351 (was 8357) 2024-11-13T07:34:07,206 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-13T07:34:07,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T07:34:07,206 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:34:07,206 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:34:07,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T07:34:07,207 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T07:34:07,207 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=269611031, stopped=false 2024-11-13T07:34:07,207 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2998b885bdef,44967,1731483205524 2024-11-13T07:34:07,230 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:34:07,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:34:07,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:34:07,230 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:34:07,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:34:07,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:34:07,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T07:34:07,230 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:34:07,230 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:34:07,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:34:07,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:34:07,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:34:07,231 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,39583,1731483206286' ***** 2024-11-13T07:34:07,231 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T07:34:07,231 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,37319,1731483206386' ***** 2024-11-13T07:34:07,231 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T07:34:07,231 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T07:34:07,232 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(959): stopping server 2998b885bdef,39583,1731483206286 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(3091): Received CLOSE for 47901928f170b169c8ff6f7dea663386 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2998b885bdef:39583. 2024-11-13T07:34:07,232 DEBUG [RS:0;2998b885bdef:39583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:34:07,232 DEBUG [RS:0;2998b885bdef:39583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,232 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(959): stopping server 2998b885bdef,37319,1731483206386 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2998b885bdef:37319. 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:34:07,232 DEBUG [RS:1;2998b885bdef:37319 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:34:07,232 DEBUG [RS:1;2998b885bdef:37319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,232 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T07:34:07,232 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T07:34:07,232 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 47901928f170b169c8ff6f7dea663386, disabling compactions & flushes 2024-11-13T07:34:07,232 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1325): Online Regions={47901928f170b169c8ff6f7dea663386=testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386.} 2024-11-13T07:34:07,232 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:34:07,232 DEBUG [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1351): Waiting on 47901928f170b169c8ff6f7dea663386 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:34:07,233 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. after waiting 0 ms 2024-11-13T07:34:07,233 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:34:07,233 DEBUG [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T07:34:07,233 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:34:07,233 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:34:07,233 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-13T07:34:07,237 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/info/12b0b66d282746689b095b5fdf7b24fe is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386./info:regioninfo/1731483226076/Put/seqid=0 2024-11-13T07:34:07,239 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/47901928f170b169c8ff6f7dea663386/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T07:34:07,240 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:34:07,240 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 47901928f170b169c8ff6f7dea663386: Waiting for close lock at 1731483247232Running coprocessor pre-close hooks at 1731483247232Disabling compacts and flushes for region at 1731483247232Disabling writes for close at 1731483247233 (+1 ms)Writing region close event to WAL at 1731483247236 (+3 ms)Running coprocessor post-close hooks at 1731483247240 (+4 ms)Closed at 1731483247240 2024-11-13T07:34:07,240 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731483221728.47901928f170b169c8ff6f7dea663386. 2024-11-13T07:34:07,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741943_1123 (size=6778) 2024-11-13T07:34:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741943_1123 (size=6778) 2024-11-13T07:34:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741943_1123 (size=6778) 2024-11-13T07:34:07,245 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/info/12b0b66d282746689b095b5fdf7b24fe 2024-11-13T07:34:07,250 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/.tmp/info/12b0b66d282746689b095b5fdf7b24fe as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/info/12b0b66d282746689b095b5fdf7b24fe 2024-11-13T07:34:07,256 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/info/12b0b66d282746689b095b5fdf7b24fe, entries=8, sequenceid=23, filesize=6.6 K 2024-11-13T07:34:07,257 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-13T07:34:07,257 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T07:34:07,262 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-13T07:34:07,263 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T07:34:07,263 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:34:07,263 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731483247233Running coprocessor pre-close hooks at 1731483247233Disabling compacts and flushes for region at 1731483247233Disabling writes for close at 1731483247233Obtaining lock to block concurrent updates at 1731483247233Preparing flush snapshotting stores in 1588230740 at 1731483247233Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731483247233Flushing stores of hbase:meta,,1.1588230740 at 1731483247234 (+1 ms)Flushing 1588230740/info: creating writer at 1731483247234Flushing 1588230740/info: appending metadata at 1731483247237 (+3 ms)Flushing 1588230740/info: closing flushed file at 1731483247237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c9a71b3: reopening flushed file at 1731483247250 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1731483247257 (+7 ms)Writing region close event to WAL at 1731483247259 (+2 ms)Running coprocessor post-close hooks at 1731483247263 (+4 ms)Closed at 1731483247263 2024-11-13T07:34:07,263 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T07:34:07,267 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:34:07,267 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:34:07,433 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,37319,1731483206386; all regions closed. 2024-11-13T07:34:07,433 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,39583,1731483206286; all regions closed. 2024-11-13T07:34:07,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741834_1010 (size=1032) 2024-11-13T07:34:07,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741834_1010 (size=1032) 2024-11-13T07:34:07,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741893_1071 (size=2625) 2024-11-13T07:34:07,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741834_1010 (size=1032) 2024-11-13T07:34:07,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741893_1071 (size=2625) 2024-11-13T07:34:07,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741893_1071 (size=2625) 2024-11-13T07:34:07,443 DEBUG [RS:1;2998b885bdef:37319 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs 2024-11-13T07:34:07,443 DEBUG [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C37319%2C1731483206386:(num 1731483207992) 2024-11-13T07:34:07,443 INFO [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C39583%2C1731483206286.meta:.meta(num 1731483224928) 2024-11-13T07:34:07,443 DEBUG [RS:1;2998b885bdef:37319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:34:07,443 INFO [RS:1;2998b885bdef:37319 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:34:07,443 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:34:07,444 INFO [RS:1;2998b885bdef:37319 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:34:07,444 INFO [RS:1;2998b885bdef:37319 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37319 2024-11-13T07:34:07,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741833_1009 (size=2764) 2024-11-13T07:34:07,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741833_1009 (size=2764) 2024-11-13T07:34:07,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741833_1009 (size=2764) 2024-11-13T07:34:07,448 DEBUG [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/oldWALs 2024-11-13T07:34:07,448 INFO [RS:0;2998b885bdef:39583 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C39583%2C1731483206286:(num 1731483207992) 2024-11-13T07:34:07,448 DEBUG [RS:0;2998b885bdef:39583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:34:07,448 INFO [RS:0;2998b885bdef:39583 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:34:07,448 INFO [RS:0;2998b885bdef:39583 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:34:07,448 INFO [RS:0;2998b885bdef:39583 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T07:34:07,448 INFO [RS:0;2998b885bdef:39583 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:34:07,448 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:34:07,448 INFO [RS:0;2998b885bdef:39583 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39583 2024-11-13T07:34:07,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:34:07,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,37319,1731483206386 2024-11-13T07:34:07,455 INFO [RS:1;2998b885bdef:37319 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:34:07,465 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,39583,1731483206286 2024-11-13T07:34:07,465 INFO [RS:0;2998b885bdef:39583 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:34:07,476 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,37319,1731483206386] 2024-11-13T07:34:07,497 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,37319,1731483206386 already deleted, retry=false 2024-11-13T07:34:07,497 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2998b885bdef,37319,1731483206386 expired; onlineServers=1 2024-11-13T07:34:07,497 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,39583,1731483206286] 2024-11-13T07:34:07,508 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,39583,1731483206286 already deleted, retry=false 2024-11-13T07:34:07,508 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2998b885bdef,39583,1731483206286 expired; onlineServers=0 2024-11-13T07:34:07,508 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2998b885bdef,44967,1731483205524' ***** 2024-11-13T07:34:07,508 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T07:34:07,508 INFO [M:0;2998b885bdef:44967 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:34:07,509 INFO [M:0;2998b885bdef:44967 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:34:07,509 DEBUG [M:0;2998b885bdef:44967 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T07:34:07,509 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T07:34:07,509 DEBUG [M:0;2998b885bdef:44967 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T07:34:07,509 DEBUG [master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731483207677 {}] cleaner.HFileCleaner(306): Exit Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731483207677,5,FailOnTimeoutGroup] 2024-11-13T07:34:07,509 DEBUG [master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731483207676 {}] cleaner.HFileCleaner(306): Exit Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731483207676,5,FailOnTimeoutGroup] 2024-11-13T07:34:07,510 INFO [M:0;2998b885bdef:44967 {}] hbase.ChoreService(370): Chore service for: master/2998b885bdef:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T07:34:07,510 INFO [M:0;2998b885bdef:44967 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:34:07,511 DEBUG [M:0;2998b885bdef:44967 {}] master.HMaster(1795): Stopping service threads 2024-11-13T07:34:07,511 INFO [M:0;2998b885bdef:44967 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T07:34:07,511 INFO [M:0;2998b885bdef:44967 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T07:34:07,512 INFO [M:0;2998b885bdef:44967 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T07:34:07,512 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T07:34:07,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T07:34:07,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:34:07,519 DEBUG [M:0;2998b885bdef:44967 {}] zookeeper.ZKUtil(347): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T07:34:07,519 WARN [M:0;2998b885bdef:44967 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T07:34:07,520 INFO [M:0;2998b885bdef:44967 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/.lastflushedseqids 2024-11-13T07:34:07,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741944_1124 (size=119) 2024-11-13T07:34:07,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741944_1124 (size=119) 2024-11-13T07:34:07,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741944_1124 (size=119) 2024-11-13T07:34:07,535 INFO [M:0;2998b885bdef:44967 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T07:34:07,535 INFO [M:0;2998b885bdef:44967 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T07:34:07,535 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T07:34:07,535 INFO [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:34:07,535 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:34:07,535 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T07:34:07,535 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:34:07,536 INFO [M:0;2998b885bdef:44967 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-13T07:34:07,552 DEBUG [M:0;2998b885bdef:44967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09887193b3ff4bf981aca4f9a12a2fdb is 82, key is hbase:meta,,1/info:regioninfo/1731483225278/Put/seqid=0 2024-11-13T07:34:07,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741945_1125 (size=6063) 2024-11-13T07:34:07,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741945_1125 (size=6063) 2024-11-13T07:34:07,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741945_1125 (size=6063) 2024-11-13T07:34:07,558 INFO [M:0;2998b885bdef:44967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09887193b3ff4bf981aca4f9a12a2fdb 2024-11-13T07:34:07,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:34:07,576 INFO [RS:1;2998b885bdef:37319 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:34:07,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37319-0x101331cf8110002, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:34:07,576 INFO [RS:1;2998b885bdef:37319 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,37319,1731483206386; zookeeper connection closed. 2024-11-13T07:34:07,577 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31cbd4d0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31cbd4d0 2024-11-13T07:34:07,579 DEBUG [M:0;2998b885bdef:44967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68976241f2e34687bf0b57f88bf8aebe is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731483222160/Put/seqid=0 2024-11-13T07:34:07,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741946_1126 (size=7907) 2024-11-13T07:34:07,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741946_1126 (size=7907) 2024-11-13T07:34:07,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741946_1126 (size=7907) 2024-11-13T07:34:07,585 INFO [M:0;2998b885bdef:44967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68976241f2e34687bf0b57f88bf8aebe 2024-11-13T07:34:07,587 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:34:07,587 INFO [RS:0;2998b885bdef:39583 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:34:07,587 INFO [RS:0;2998b885bdef:39583 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,39583,1731483206286; zookeeper connection closed. 2024-11-13T07:34:07,587 DEBUG [pool-64-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39583-0x101331cf8110001, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:34:07,587 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@509a97ed {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@509a97ed 2024-11-13T07:34:07,588 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T07:34:07,589 INFO [M:0;2998b885bdef:44967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68976241f2e34687bf0b57f88bf8aebe 2024-11-13T07:34:07,603 DEBUG [M:0;2998b885bdef:44967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e39dcefaf1c24c699d139c75838c62aa is 69, key is 2998b885bdef,37319,1731483206386/rs:state/1731483207690/Put/seqid=0 2024-11-13T07:34:07,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741947_1127 (size=5445) 2024-11-13T07:34:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741947_1127 (size=5445) 2024-11-13T07:34:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741947_1127 (size=5445) 2024-11-13T07:34:07,609 INFO [M:0;2998b885bdef:44967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e39dcefaf1c24c699d139c75838c62aa 2024-11-13T07:34:07,613 INFO [M:0;2998b885bdef:44967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e39dcefaf1c24c699d139c75838c62aa 2024-11-13T07:34:07,614 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09887193b3ff4bf981aca4f9a12a2fdb as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09887193b3ff4bf981aca4f9a12a2fdb 2024-11-13T07:34:07,618 INFO [M:0;2998b885bdef:44967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09887193b3ff4bf981aca4f9a12a2fdb, entries=14, sequenceid=207, filesize=5.9 K 2024-11-13T07:34:07,619 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68976241f2e34687bf0b57f88bf8aebe as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/68976241f2e34687bf0b57f88bf8aebe 2024-11-13T07:34:07,624 INFO [M:0;2998b885bdef:44967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68976241f2e34687bf0b57f88bf8aebe 2024-11-13T07:34:07,624 INFO [M:0;2998b885bdef:44967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/68976241f2e34687bf0b57f88bf8aebe, entries=21, sequenceid=207, filesize=7.7 K 2024-11-13T07:34:07,625 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e39dcefaf1c24c699d139c75838c62aa as hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e39dcefaf1c24c699d139c75838c62aa 2024-11-13T07:34:07,629 INFO [M:0;2998b885bdef:44967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e39dcefaf1c24c699d139c75838c62aa 2024-11-13T07:34:07,629 INFO [M:0;2998b885bdef:44967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35647/user/jenkins/test-data/0e43150c-9f19-96e6-892b-0113be207f7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e39dcefaf1c24c699d139c75838c62aa, entries=3, sequenceid=207, filesize=5.3 K 2024-11-13T07:34:07,630 INFO [M:0;2998b885bdef:44967 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=207, compaction requested=false 2024-11-13T07:34:07,631 INFO [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:34:07,631 DEBUG [M:0;2998b885bdef:44967 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731483247535Disabling compacts and flushes for region at 1731483247535Disabling writes for close at 1731483247535Obtaining lock to block concurrent updates at 1731483247536 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731483247536Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1731483247536Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731483247536Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731483247537 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731483247552 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731483247552Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731483247563 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731483247578 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731483247578Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731483247589 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731483247602 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731483247602Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e3d8479: reopening flushed file at 1731483247613 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6970c339: reopening flushed file at 1731483247619 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21741c75: reopening flushed file at 1731483247624 (+5 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 95ms, sequenceid=207, compaction requested=false at 1731483247630 (+6 ms)Writing region close event to WAL at 1731483247631 (+1 ms)Closed at 1731483247631 2024-11-13T07:34:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35659 is added to blk_1073741830_1006 (size=102162) 2024-11-13T07:34:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41959 is added to blk_1073741830_1006 (size=102162) 2024-11-13T07:34:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38619 is added to blk_1073741830_1006 (size=102162) 2024-11-13T07:34:07,634 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:34:07,634 INFO [M:0;2998b885bdef:44967 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T07:34:07,634 INFO [M:0;2998b885bdef:44967 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44967 2024-11-13T07:34:07,634 INFO [M:0;2998b885bdef:44967 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:34:07,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:34:07,745 INFO [M:0;2998b885bdef:44967 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:34:07,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44967-0x101331cf8110000, quorum=127.0.0.1:65209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:34:07,751 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483236463 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483236463 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731483227138/wal.1731483236463 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T07:34:07,753 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226947 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226947 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731483226500/wal.1731483226947 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T07:34:07,754 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731483217218/wal.1731483217338 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:07,756 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483237243 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483237243 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731483236686/wal.1731483237243 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T07:34:07,757 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731483237427/wal.1731483237510 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:07,757 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731483242006/wal.1731483242270 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:07,758 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731483210315/wal.1731483210409 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:34:07,760 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal.1731483210142 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal.1731483210142 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731483209759/wal.1731483210142 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T07:34:07,763 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal.1731483209426 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal.1731483209426 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731483208972/wal.1731483209426 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1484725256_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T07:34:07,767 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:34:07,769 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:34:07,769 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:34:07,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:34:07,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,STOPPED} 2024-11-13T07:34:07,772 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:34:07,772 WARN [BP-951039070-172.17.0.3-1731483201138 heartbeating to localhost/127.0.0.1:35647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:34:07,772 WARN [BP-951039070-172.17.0.3-1731483201138 heartbeating to localhost/127.0.0.1:35647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-951039070-172.17.0.3-1731483201138 (Datanode Uuid afbb5454-2714-4ec2-969e-e2935062c7ce) service to localhost/127.0.0.1:35647 2024-11-13T07:34:07,772 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:34:07,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data5/current/BP-951039070-172.17.0.3-1731483201138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:34:07,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data6/current/BP-951039070-172.17.0.3-1731483201138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:34:07,774 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:34:07,775 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:34:07,776 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:34:07,776 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:34:07,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:34:07,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,STOPPED} 2024-11-13T07:34:07,777 WARN [BP-951039070-172.17.0.3-1731483201138 heartbeating to localhost/127.0.0.1:35647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:34:07,777 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:34:07,777 WARN [BP-951039070-172.17.0.3-1731483201138 heartbeating to localhost/127.0.0.1:35647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-951039070-172.17.0.3-1731483201138 (Datanode Uuid f0ea07c0-a733-458f-ac7e-ccd7994cd7fd) service to localhost/127.0.0.1:35647 2024-11-13T07:34:07,777 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:34:07,778 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data3/current/BP-951039070-172.17.0.3-1731483201138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:34:07,778 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data4/current/BP-951039070-172.17.0.3-1731483201138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:34:07,778 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:34:07,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:34:07,781 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:34:07,781 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:34:07,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:34:07,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,STOPPED} 2024-11-13T07:34:07,782 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:34:07,782 WARN [BP-951039070-172.17.0.3-1731483201138 heartbeating to localhost/127.0.0.1:35647 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:34:07,782 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:34:07,782 WARN [BP-951039070-172.17.0.3-1731483201138 heartbeating to localhost/127.0.0.1:35647 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-951039070-172.17.0.3-1731483201138 (Datanode Uuid f073f10b-1213-44e7-916a-81d6e3bf0107) service to localhost/127.0.0.1:35647 2024-11-13T07:34:07,783 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data1/current/BP-951039070-172.17.0.3-1731483201138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:34:07,783 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/cluster_443fba65-8be6-77f0-7db3-b74c16a49775/data/data2/current/BP-951039070-172.17.0.3-1731483201138 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:34:07,784 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:34:07,789 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T07:34:07,789 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:34:07,789 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:34:07,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:34:07,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a2edd5e1-cf06-ee49-5074-9f9c13664399/hadoop.log.dir/,STOPPED} 2024-11-13T07:34:07,798 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T07:34:07,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down