2024-11-24 04:48:17,451 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-11-24 04:48:17,468 main DEBUG Took 0.014565 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 04:48:17,468 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 04:48:17,469 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 04:48:17,469 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 04:48:17,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,477 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 04:48:17,492 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,495 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,496 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,497 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,498 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,499 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,501 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,501 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,502 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,503 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,504 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,505 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,505 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,507 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,508 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,508 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:48:17,509 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,509 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 04:48:17,511 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:48:17,513 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 04:48:17,515 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 04:48:17,516 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 04:48:17,517 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 04:48:17,518 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 04:48:17,528 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 04:48:17,532 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 04:48:17,534 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 04:48:17,534 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 04:48:17,535 main DEBUG createAppenders(={Console}) 2024-11-24 04:48:17,536 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 initialized 2024-11-24 04:48:17,536 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-11-24 04:48:17,536 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 OK. 2024-11-24 04:48:17,537 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 04:48:17,537 main DEBUG OutputStream closed 2024-11-24 04:48:17,537 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 04:48:17,537 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 04:48:17,538 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54a7079e OK 2024-11-24 04:48:17,608 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 04:48:17,610 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 04:48:17,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 04:48:17,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 04:48:17,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 04:48:17,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 04:48:17,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 04:48:17,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 04:48:17,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 04:48:17,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 04:48:17,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 04:48:17,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 04:48:17,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 04:48:17,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 04:48:17,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 04:48:17,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 04:48:17,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 04:48:17,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 04:48:17,619 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 04:48:17,619 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19b93fa8) with optional ClassLoader: null 2024-11-24 04:48:17,619 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 04:48:17,620 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19b93fa8] started OK. 2024-11-24T04:48:17,859 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9 2024-11-24 04:48:17,862 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 04:48:17,863 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T04:48:17,871 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T04:48:17,876 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-24T04:48:17,900 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T04:48:17,942 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T04:48:17,942 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T04:48:17,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T04:48:17,968 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672, deleteOnExit=true 2024-11-24T04:48:17,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T04:48:17,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/test.cache.data in system properties and HBase conf 2024-11-24T04:48:17,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T04:48:17,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir in system properties and HBase conf 2024-11-24T04:48:17,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T04:48:17,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T04:48:17,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T04:48:18,050 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T04:48:18,139 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T04:48:18,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:48:18,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:48:18,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T04:48:18,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:48:18,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T04:48:18,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T04:48:18,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:48:18,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:48:18,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T04:48:18,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/nfs.dump.dir in system properties and HBase conf 2024-11-24T04:48:18,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/java.io.tmpdir in system properties and HBase conf 2024-11-24T04:48:18,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:48:18,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T04:48:18,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T04:48:19,283 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T04:48:19,357 INFO [Time-limited test {}] log.Log(170): Logging initialized @2989ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T04:48:19,432 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:48:19,500 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:48:19,528 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:48:19,528 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:48:19,529 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:48:19,545 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:48:19,550 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:48:19,551 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:48:19,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14f32243{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/java.io.tmpdir/jetty-localhost-34079-hadoop-hdfs-3_4_1-tests_jar-_-any-13268241884636597465/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:48:19,744 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:34079} 2024-11-24T04:48:19,744 INFO [Time-limited test {}] server.Server(415): Started @3376ms 2024-11-24T04:48:20,276 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:48:20,285 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:48:20,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:48:20,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:48:20,286 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:48:20,287 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:48:20,287 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:48:20,393 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d480c14{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/java.io.tmpdir/jetty-localhost-34221-hadoop-hdfs-3_4_1-tests_jar-_-any-12880574440297053639/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:48:20,393 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:34221} 2024-11-24T04:48:20,394 INFO [Time-limited test {}] server.Server(415): Started @4025ms 2024-11-24T04:48:20,455 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:48:20,563 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:48:20,569 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:48:20,570 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:48:20,571 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:48:20,571 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:48:20,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:48:20,573 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:48:20,709 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75d96791{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/java.io.tmpdir/jetty-localhost-46357-hadoop-hdfs-3_4_1-tests_jar-_-any-8532430923838432062/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:48:20,710 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:46357} 2024-11-24T04:48:20,710 INFO [Time-limited test {}] server.Server(415): Started @4342ms 2024-11-24T04:48:20,713 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:48:20,747 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:48:20,752 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:48:20,755 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:48:20,755 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:48:20,755 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:48:20,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:48:20,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:48:20,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@773b4a40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/java.io.tmpdir/jetty-localhost-41859-hadoop-hdfs-3_4_1-tests_jar-_-any-17924341635416738149/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:48:20,865 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:41859} 2024-11-24T04:48:20,866 INFO [Time-limited test {}] server.Server(415): Started @4497ms 2024-11-24T04:48:20,870 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:48:22,427 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data1/current/BP-2110872418-172.17.0.2-1732423698697/current, will proceed with Du for space computation calculation, 2024-11-24T04:48:22,427 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data4/current/BP-2110872418-172.17.0.2-1732423698697/current, will proceed with Du for space computation calculation, 2024-11-24T04:48:22,427 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data3/current/BP-2110872418-172.17.0.2-1732423698697/current, will proceed with Du for space computation calculation, 2024-11-24T04:48:22,427 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data2/current/BP-2110872418-172.17.0.2-1732423698697/current, will proceed with Du for space computation calculation, 2024-11-24T04:48:22,456 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:48:22,456 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:48:22,471 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data6/current/BP-2110872418-172.17.0.2-1732423698697/current, will proceed with Du for space computation calculation, 2024-11-24T04:48:22,471 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data5/current/BP-2110872418-172.17.0.2-1732423698697/current, will proceed with Du for space computation calculation, 2024-11-24T04:48:22,491 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:48:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf69f058c4a4ebc04 with lease ID 0xe0e618139a00a8dc: Processing first storage report for DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad from datanode DatanodeRegistration(127.0.0.1:35959, datanodeUuid=5230f75c-0cac-461f-9046-29cca0650db0, infoPort=44929, infoSecurePort=0, ipcPort=40807, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697) 2024-11-24T04:48:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf69f058c4a4ebc04 with lease ID 0xe0e618139a00a8dc: from storage DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad node DatanodeRegistration(127.0.0.1:35959, datanodeUuid=5230f75c-0cac-461f-9046-29cca0650db0, infoPort=44929, infoSecurePort=0, ipcPort=40807, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:48:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeca7afca8d98f6bf with lease ID 0xe0e618139a00a8dd: Processing first storage report for DS-859f7789-1b26-4152-ac45-6d8c026b65ca from datanode DatanodeRegistration(127.0.0.1:43379, datanodeUuid=152bafc3-db70-4c2a-8428-6f5a414ad0e5, infoPort=44569, infoSecurePort=0, ipcPort=36887, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697) 2024-11-24T04:48:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeca7afca8d98f6bf with lease ID 0xe0e618139a00a8dd: from storage DS-859f7789-1b26-4152-ac45-6d8c026b65ca node DatanodeRegistration(127.0.0.1:43379, datanodeUuid=152bafc3-db70-4c2a-8428-6f5a414ad0e5, infoPort=44569, infoSecurePort=0, ipcPort=36887, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:48:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8bf34991da8229f with lease ID 0xe0e618139a00a8de: Processing first storage report for DS-e5725c27-fd79-47c0-a49e-0cea0536f84b from datanode DatanodeRegistration(127.0.0.1:45441, datanodeUuid=89350da1-96f6-4e8b-bcd1-0f12b66d8f37, infoPort=45051, infoSecurePort=0, ipcPort=37235, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697) 2024-11-24T04:48:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8bf34991da8229f with lease ID 0xe0e618139a00a8de: from storage DS-e5725c27-fd79-47c0-a49e-0cea0536f84b node DatanodeRegistration(127.0.0.1:45441, datanodeUuid=89350da1-96f6-4e8b-bcd1-0f12b66d8f37, infoPort=45051, infoSecurePort=0, ipcPort=37235, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:48:22,507 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf69f058c4a4ebc04 with lease ID 0xe0e618139a00a8dc: Processing first storage report for DS-10e0f14e-b4f3-4d15-829c-7139537021d5 from datanode DatanodeRegistration(127.0.0.1:35959, datanodeUuid=5230f75c-0cac-461f-9046-29cca0650db0, infoPort=44929, infoSecurePort=0, ipcPort=40807, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697) 2024-11-24T04:48:22,507 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf69f058c4a4ebc04 with lease ID 0xe0e618139a00a8dc: from storage DS-10e0f14e-b4f3-4d15-829c-7139537021d5 node DatanodeRegistration(127.0.0.1:35959, datanodeUuid=5230f75c-0cac-461f-9046-29cca0650db0, infoPort=44929, infoSecurePort=0, ipcPort=40807, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:48:22,507 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeca7afca8d98f6bf with lease ID 0xe0e618139a00a8dd: Processing first storage report for DS-5fdcbd22-9150-4d71-a64d-e13092eb3bb4 from datanode DatanodeRegistration(127.0.0.1:43379, datanodeUuid=152bafc3-db70-4c2a-8428-6f5a414ad0e5, infoPort=44569, infoSecurePort=0, ipcPort=36887, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697) 2024-11-24T04:48:22,507 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeca7afca8d98f6bf with lease ID 0xe0e618139a00a8dd: from storage DS-5fdcbd22-9150-4d71-a64d-e13092eb3bb4 node DatanodeRegistration(127.0.0.1:43379, datanodeUuid=152bafc3-db70-4c2a-8428-6f5a414ad0e5, infoPort=44569, infoSecurePort=0, ipcPort=36887, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:48:22,507 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8bf34991da8229f with lease ID 0xe0e618139a00a8de: Processing first storage report for DS-0bcef851-fff8-4963-99d9-b8de1fd4896b from datanode DatanodeRegistration(127.0.0.1:45441, datanodeUuid=89350da1-96f6-4e8b-bcd1-0f12b66d8f37, infoPort=45051, infoSecurePort=0, ipcPort=37235, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697) 2024-11-24T04:48:22,507 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8bf34991da8229f with lease ID 0xe0e618139a00a8de: from storage DS-0bcef851-fff8-4963-99d9-b8de1fd4896b node DatanodeRegistration(127.0.0.1:45441, datanodeUuid=89350da1-96f6-4e8b-bcd1-0f12b66d8f37, infoPort=45051, infoSecurePort=0, ipcPort=37235, storageInfo=lv=-57;cid=testClusterID;nsid=1117573912;c=1732423698697), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:48:22,528 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9 2024-11-24T04:48:22,593 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/zookeeper_0, clientPort=57729, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T04:48:22,613 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57729 2024-11-24T04:48:22,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:22,634 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:22,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:48:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:48:22,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:48:23,250 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269 with version=8 2024-11-24T04:48:23,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/hbase-staging 2024-11-24T04:48:23,506 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:48:23,515 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:23,515 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:23,519 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:48:23,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:23,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:48:23,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T04:48:23,707 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T04:48:23,718 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T04:48:23,723 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:48:23,752 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 49589 (auto-detected) 2024-11-24T04:48:23,753 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T04:48:23,771 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38057 2024-11-24T04:48:23,789 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38057 connecting to ZooKeeper ensemble=127.0.0.1:57729 2024-11-24T04:48:23,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380570x0, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:48:23,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38057-0x1016b2bd3dc0000 connected 2024-11-24T04:48:23,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:23,992 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:48:24,008 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269, hbase.cluster.distributed=false 2024-11-24T04:48:24,028 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:48:24,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38057 2024-11-24T04:48:24,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38057 2024-11-24T04:48:24,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38057 2024-11-24T04:48:24,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38057 2024-11-24T04:48:24,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38057 2024-11-24T04:48:24,125 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:48:24,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,127 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:48:24,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:48:24,129 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:48:24,132 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:48:24,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42899 2024-11-24T04:48:24,134 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42899 connecting to ZooKeeper ensemble=127.0.0.1:57729 2024-11-24T04:48:24,135 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,138 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428990x0, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:48:24,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428990x0, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:48:24,149 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42899-0x1016b2bd3dc0001 connected 2024-11-24T04:48:24,152 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:48:24,159 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:48:24,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:48:24,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:48:24,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42899 2024-11-24T04:48:24,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42899 2024-11-24T04:48:24,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42899 2024-11-24T04:48:24,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42899 2024-11-24T04:48:24,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42899 2024-11-24T04:48:24,184 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:48:24,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,185 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:48:24,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:48:24,185 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:48:24,186 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:48:24,186 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44211 2024-11-24T04:48:24,188 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44211 connecting to ZooKeeper ensemble=127.0.0.1:57729 2024-11-24T04:48:24,189 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442110x0, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:48:24,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442110x0, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:48:24,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44211-0x1016b2bd3dc0002 connected 2024-11-24T04:48:24,202 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:48:24,202 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:48:24,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:48:24,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:48:24,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44211 2024-11-24T04:48:24,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44211 2024-11-24T04:48:24,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44211 2024-11-24T04:48:24,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44211 2024-11-24T04:48:24,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44211 2024-11-24T04:48:24,223 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:48:24,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,223 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,223 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:48:24,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:48:24,224 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:48:24,224 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:48:24,224 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:48:24,225 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33335 2024-11-24T04:48:24,226 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33335 connecting to ZooKeeper ensemble=127.0.0.1:57729 2024-11-24T04:48:24,228 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,230 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333350x0, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:48:24,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:333350x0, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:48:24,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33335-0x1016b2bd3dc0003 connected 2024-11-24T04:48:24,244 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:48:24,245 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:48:24,246 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:48:24,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:48:24,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33335 2024-11-24T04:48:24,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33335 2024-11-24T04:48:24,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33335 2024-11-24T04:48:24,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33335 2024-11-24T04:48:24,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33335 2024-11-24T04:48:24,268 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4464c5b832df:38057 2024-11-24T04:48:24,269 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4464c5b832df,38057,1732423703360 2024-11-24T04:48:24,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,287 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4464c5b832df,38057,1732423703360 2024-11-24T04:48:24,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:48:24,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:48:24,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:48:24,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,379 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T04:48:24,380 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4464c5b832df,38057,1732423703360 from backup master directory 2024-11-24T04:48:24,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4464c5b832df,38057,1732423703360 2024-11-24T04:48:24,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:48:24,480 WARN [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:48:24,480 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4464c5b832df,38057,1732423703360 2024-11-24T04:48:24,483 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T04:48:24,485 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T04:48:24,536 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/hbase.id] with ID: 894908be-bd12-4c70-96f1-bb976b7347b7 2024-11-24T04:48:24,537 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/.tmp/hbase.id 2024-11-24T04:48:24,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:48:24,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:48:24,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:48:24,552 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/.tmp/hbase.id]:[hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/hbase.id] 2024-11-24T04:48:24,597 INFO [master/4464c5b832df:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:48:24,601 INFO [master/4464c5b832df:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T04:48:24,619 INFO [master/4464c5b832df:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-24T04:48:24,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:48:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:48:24,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:48:24,683 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:48:24,685 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T04:48:24,691 INFO [master/4464c5b832df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:24,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:48:24,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:48:24,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:48:24,736 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store 2024-11-24T04:48:24,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:48:24,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:48:24,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:48:24,761 INFO [master/4464c5b832df:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T04:48:24,765 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:24,766 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:48:24,767 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:48:24,767 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:48:24,769 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:48:24,769 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:48:24,769 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:48:24,771 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732423704766Disabling compacts and flushes for region at 1732423704766Disabling writes for close at 1732423704769 (+3 ms)Writing region close event to WAL at 1732423704769Closed at 1732423704769 2024-11-24T04:48:24,773 WARN [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/.initializing 2024-11-24T04:48:24,774 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/WALs/4464c5b832df,38057,1732423703360 2024-11-24T04:48:24,783 INFO [master/4464c5b832df:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:24,801 INFO [master/4464c5b832df:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C38057%2C1732423703360, suffix=, logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/WALs/4464c5b832df,38057,1732423703360, archiveDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/oldWALs, maxLogs=10 2024-11-24T04:48:24,827 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/WALs/4464c5b832df,38057,1732423703360/4464c5b832df%2C38057%2C1732423703360.1732423704806, exclude list is [], retry=0 2024-11-24T04:48:24,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:24,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:24,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:24,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:24,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T04:48:24,881 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-24T04:48:24,939 INFO [master/4464c5b832df:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/WALs/4464c5b832df,38057,1732423703360/4464c5b832df%2C38057%2C1732423703360.1732423704806 2024-11-24T04:48:24,940 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:24,940 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:24,941 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:24,943 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:24,944 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:24,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:24,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T04:48:25,002 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:25,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T04:48:25,009 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,010 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:25,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T04:48:25,014 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:25,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T04:48:25,019 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,020 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:25,020 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,025 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,026 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,032 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,033 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,036 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:48:25,041 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:48:25,045 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:25,046 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70922236, jitterRate=0.056823670864105225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:48:25,053 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732423704955Initializing all the Stores at 1732423704957 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423704957Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423704958 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423704958Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423704958Cleaning up temporary data from old regions at 1732423705033 (+75 ms)Region opened successfully at 1732423705053 (+20 ms) 2024-11-24T04:48:25,055 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T04:48:25,086 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b6a3a00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:48:25,115 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T04:48:25,125 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T04:48:25,126 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T04:48:25,128 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T04:48:25,130 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T04:48:25,134 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T04:48:25,134 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T04:48:25,158 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T04:48:25,166 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T04:48:25,221 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T04:48:25,223 INFO [master/4464c5b832df:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T04:48:25,225 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T04:48:25,232 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T04:48:25,234 INFO [master/4464c5b832df:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T04:48:25,237 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T04:48:25,242 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T04:48:25,244 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T04:48:25,253 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T04:48:25,272 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T04:48:25,285 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:48:25,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,298 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4464c5b832df,38057,1732423703360, sessionid=0x1016b2bd3dc0000, setting cluster-up flag (Was=false) 2024-11-24T04:48:25,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,359 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T04:48:25,363 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4464c5b832df,38057,1732423703360 2024-11-24T04:48:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:25,422 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T04:48:25,426 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4464c5b832df,38057,1732423703360 2024-11-24T04:48:25,438 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T04:48:25,458 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(746): ClusterId : 894908be-bd12-4c70-96f1-bb976b7347b7 2024-11-24T04:48:25,458 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(746): ClusterId : 894908be-bd12-4c70-96f1-bb976b7347b7 2024-11-24T04:48:25,458 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(746): ClusterId : 894908be-bd12-4c70-96f1-bb976b7347b7 2024-11-24T04:48:25,460 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:48:25,460 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:48:25,460 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:48:25,476 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:48:25,476 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:48:25,476 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:48:25,476 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:48:25,476 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:48:25,476 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:48:25,487 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:48:25,487 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:48:25,487 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:48:25,487 DEBUG [RS:2;4464c5b832df:33335 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65be27ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:48:25,487 DEBUG [RS:1;4464c5b832df:44211 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582c4702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:48:25,487 DEBUG [RS:0;4464c5b832df:42899 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2e5ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:48:25,503 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4464c5b832df:33335 2024-11-24T04:48:25,505 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4464c5b832df:44211 2024-11-24T04:48:25,506 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4464c5b832df:42899 2024-11-24T04:48:25,507 INFO [RS:2;4464c5b832df:33335 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:48:25,507 INFO [RS:0;4464c5b832df:42899 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:48:25,507 INFO [RS:2;4464c5b832df:33335 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:48:25,507 INFO [RS:0;4464c5b832df:42899 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:48:25,507 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:48:25,507 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:48:25,518 INFO [RS:1;4464c5b832df:44211 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:48:25,518 INFO [RS:1;4464c5b832df:44211 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:48:25,518 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:48:25,522 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,38057,1732423703360 with port=33335, startcode=1732423704223 2024-11-24T04:48:25,522 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,38057,1732423703360 with port=44211, startcode=1732423704184 2024-11-24T04:48:25,522 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,38057,1732423703360 with port=42899, startcode=1732423704094 2024-11-24T04:48:25,534 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T04:48:25,537 DEBUG [RS:2;4464c5b832df:33335 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:48:25,537 DEBUG [RS:0;4464c5b832df:42899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:48:25,537 DEBUG [RS:1;4464c5b832df:44211 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:48:25,546 INFO [master/4464c5b832df:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T04:48:25,555 INFO [master/4464c5b832df:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T04:48:25,562 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4464c5b832df,38057,1732423703360 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T04:48:25,571 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:48:25,571 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:48:25,572 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:48:25,572 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:48:25,572 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4464c5b832df:0, corePoolSize=10, maxPoolSize=10 2024-11-24T04:48:25,573 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,573 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:48:25,573 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,590 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42115, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:48:25,590 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36657, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:48:25,590 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41621, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:48:25,594 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:48:25,595 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T04:48:25,600 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T04:48:25,601 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732423735601 2024-11-24T04:48:25,602 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,602 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T04:48:25,603 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T04:48:25,605 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T04:48:25,610 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T04:48:25,610 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T04:48:25,610 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T04:48:25,611 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T04:48:25,613 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T04:48:25,614 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T04:48:25,620 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,633 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T04:48:25,634 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T04:48:25,634 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T04:48:25,637 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T04:48:25,637 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T04:48:25,641 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:48:25,641 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:48:25,641 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T04:48:25,641 WARN [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:48:25,641 WARN [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:48:25,641 WARN [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T04:48:25,673 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.large.0-1732423705673,5,FailOnTimeoutGroup] 2024-11-24T04:48:25,674 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.small.0-1732423705673,5,FailOnTimeoutGroup] 2024-11-24T04:48:25,674 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,674 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T04:48:25,675 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,675 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:48:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:48:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:48:25,679 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T04:48:25,679 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269 2024-11-24T04:48:25,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:48:25,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:48:25,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:48:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:25,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:48:25,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:48:25,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:25,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:48:25,704 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:48:25,704 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:25,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:48:25,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:48:25,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:25,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:48:25,712 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:48:25,712 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:25,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:25,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:48:25,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740 2024-11-24T04:48:25,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740 2024-11-24T04:48:25,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:48:25,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:48:25,719 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:48:25,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:48:25,726 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:25,727 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64481735, jitterRate=-0.0391472727060318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:48:25,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732423705695Initializing all the Stores at 1732423705697 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423705697Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423705697Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423705697Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423705697Cleaning up temporary data from old regions at 1732423705718 (+21 ms)Region opened successfully at 1732423705731 (+13 ms) 2024-11-24T04:48:25,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:48:25,731 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:48:25,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:48:25,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:48:25,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:48:25,733 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:48:25,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732423705731Disabling compacts and flushes for region at 1732423705731Disabling writes for close at 1732423705731Writing region close event to WAL at 1732423705732 (+1 ms)Closed at 1732423705732 2024-11-24T04:48:25,736 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:48:25,736 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T04:48:25,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T04:48:25,743 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,38057,1732423703360 with port=44211, startcode=1732423704184 2024-11-24T04:48:25,743 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,38057,1732423703360 with port=42899, startcode=1732423704094 2024-11-24T04:48:25,743 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,38057,1732423703360 with port=33335, startcode=1732423704223 2024-11-24T04:48:25,744 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4464c5b832df,42899,1732423704094 2024-11-24T04:48:25,747 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] master.ServerManager(517): Registering regionserver=4464c5b832df,42899,1732423704094 2024-11-24T04:48:25,752 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:48:25,755 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4464c5b832df,33335,1732423704223 2024-11-24T04:48:25,755 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] master.ServerManager(517): Registering regionserver=4464c5b832df,33335,1732423704223 2024-11-24T04:48:25,756 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269 2024-11-24T04:48:25,756 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43985 2024-11-24T04:48:25,756 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:48:25,756 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T04:48:25,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4464c5b832df,44211,1732423704184 2024-11-24T04:48:25,759 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269 2024-11-24T04:48:25,759 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43985 2024-11-24T04:48:25,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] master.ServerManager(517): Registering regionserver=4464c5b832df,44211,1732423704184 2024-11-24T04:48:25,759 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:48:25,762 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269 2024-11-24T04:48:25,762 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43985 2024-11-24T04:48:25,762 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:48:25,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:48:25,810 DEBUG [RS:0;4464c5b832df:42899 {}] zookeeper.ZKUtil(111): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4464c5b832df,42899,1732423704094 2024-11-24T04:48:25,810 WARN [RS:0;4464c5b832df:42899 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:48:25,811 INFO [RS:0;4464c5b832df:42899 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:25,811 DEBUG [RS:2;4464c5b832df:33335 {}] zookeeper.ZKUtil(111): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4464c5b832df,33335,1732423704223 2024-11-24T04:48:25,811 WARN [RS:2;4464c5b832df:33335 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:48:25,811 DEBUG [RS:1;4464c5b832df:44211 {}] zookeeper.ZKUtil(111): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4464c5b832df,44211,1732423704184 2024-11-24T04:48:25,811 WARN [RS:1;4464c5b832df:44211 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:48:25,811 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,42899,1732423704094 2024-11-24T04:48:25,811 INFO [RS:2;4464c5b832df:33335 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:25,811 INFO [RS:1;4464c5b832df:44211 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:25,811 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223 2024-11-24T04:48:25,812 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184 2024-11-24T04:48:25,813 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4464c5b832df,33335,1732423704223] 2024-11-24T04:48:25,813 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4464c5b832df,42899,1732423704094] 2024-11-24T04:48:25,814 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4464c5b832df,44211,1732423704184] 2024-11-24T04:48:25,838 INFO [RS:1;4464c5b832df:44211 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:48:25,838 INFO [RS:0;4464c5b832df:42899 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:48:25,838 INFO [RS:2;4464c5b832df:33335 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:48:25,868 INFO [RS:2;4464c5b832df:33335 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:48:25,868 INFO [RS:0;4464c5b832df:42899 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:48:25,868 INFO [RS:1;4464c5b832df:44211 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:48:25,874 INFO [RS:1;4464c5b832df:44211 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:48:25,874 INFO [RS:0;4464c5b832df:42899 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:48:25,874 INFO [RS:2;4464c5b832df:33335 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:48:25,875 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,875 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,875 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,876 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:48:25,880 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:48:25,881 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:48:25,883 INFO [RS:0;4464c5b832df:42899 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:48:25,883 INFO [RS:2;4464c5b832df:33335 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:48:25,883 INFO [RS:1;4464c5b832df:44211 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:48:25,884 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,884 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,884 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,884 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,884 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,884 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,885 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:48:25,886 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:0;4464c5b832df:42899 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:48:25,886 DEBUG [RS:1;4464c5b832df:44211 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:48:25,886 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,887 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:48:25,887 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:48:25,887 DEBUG [RS:2;4464c5b832df:33335 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:48:25,888 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,888 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,888 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,888 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,888 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,888 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,42899,1732423704094-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,44211,1732423704184-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,889 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,890 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,890 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,33335,1732423704223-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:48:25,906 WARN [4464c5b832df:38057 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T04:48:25,907 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:48:25,907 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:48:25,907 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:48:25,909 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,42899,1732423704094-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,909 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,33335,1732423704223-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,909 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,44211,1732423704184-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,909 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,909 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,909 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,909 INFO [RS:2;4464c5b832df:33335 {}] regionserver.Replication(171): 4464c5b832df,33335,1732423704223 started 2024-11-24T04:48:25,909 INFO [RS:1;4464c5b832df:44211 {}] regionserver.Replication(171): 4464c5b832df,44211,1732423704184 started 2024-11-24T04:48:25,909 INFO [RS:0;4464c5b832df:42899 {}] regionserver.Replication(171): 4464c5b832df,42899,1732423704094 started 2024-11-24T04:48:25,933 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,933 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,933 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:25,933 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1482): Serving as 4464c5b832df,42899,1732423704094, RpcServer on 4464c5b832df/172.17.0.2:42899, sessionid=0x1016b2bd3dc0001 2024-11-24T04:48:25,933 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1482): Serving as 4464c5b832df,33335,1732423704223, RpcServer on 4464c5b832df/172.17.0.2:33335, sessionid=0x1016b2bd3dc0003 2024-11-24T04:48:25,933 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1482): Serving as 4464c5b832df,44211,1732423704184, RpcServer on 4464c5b832df/172.17.0.2:44211, sessionid=0x1016b2bd3dc0002 2024-11-24T04:48:25,934 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:48:25,934 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:48:25,934 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:48:25,934 DEBUG [RS:1;4464c5b832df:44211 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4464c5b832df,44211,1732423704184 2024-11-24T04:48:25,934 DEBUG [RS:2;4464c5b832df:33335 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4464c5b832df,33335,1732423704223 2024-11-24T04:48:25,934 DEBUG [RS:0;4464c5b832df:42899 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4464c5b832df,42899,1732423704094 2024-11-24T04:48:25,934 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,33335,1732423704223' 2024-11-24T04:48:25,934 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,42899,1732423704094' 2024-11-24T04:48:25,934 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,44211,1732423704184' 2024-11-24T04:48:25,934 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:48:25,934 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:48:25,934 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:48:25,935 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:48:25,935 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:48:25,935 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:48:25,936 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:48:25,936 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:48:25,936 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:48:25,936 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:48:25,936 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:48:25,936 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:48:25,936 DEBUG [RS:0;4464c5b832df:42899 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4464c5b832df,42899,1732423704094 2024-11-24T04:48:25,936 DEBUG [RS:2;4464c5b832df:33335 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4464c5b832df,33335,1732423704223 2024-11-24T04:48:25,936 DEBUG [RS:1;4464c5b832df:44211 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4464c5b832df,44211,1732423704184 2024-11-24T04:48:25,936 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,42899,1732423704094' 2024-11-24T04:48:25,936 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,44211,1732423704184' 2024-11-24T04:48:25,936 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,33335,1732423704223' 2024-11-24T04:48:25,936 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:48:25,936 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:48:25,936 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:48:25,937 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:48:25,937 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:48:25,937 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:48:25,938 DEBUG [RS:1;4464c5b832df:44211 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:48:25,938 DEBUG [RS:0;4464c5b832df:42899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:48:25,938 DEBUG [RS:2;4464c5b832df:33335 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:48:25,938 INFO [RS:1;4464c5b832df:44211 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:48:25,938 INFO [RS:0;4464c5b832df:42899 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:48:25,938 INFO [RS:2;4464c5b832df:33335 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:48:25,938 INFO [RS:1;4464c5b832df:44211 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:48:25,938 INFO [RS:0;4464c5b832df:42899 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:48:25,938 INFO [RS:2;4464c5b832df:33335 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:48:26,044 INFO [RS:1;4464c5b832df:44211 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:26,044 INFO [RS:0;4464c5b832df:42899 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:26,044 INFO [RS:2;4464c5b832df:33335 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:26,049 INFO [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C33335%2C1732423704223, suffix=, logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223, archiveDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs, maxLogs=32 2024-11-24T04:48:26,049 INFO [RS:0;4464c5b832df:42899 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C42899%2C1732423704094, suffix=, logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,42899,1732423704094, archiveDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs, maxLogs=32 2024-11-24T04:48:26,049 INFO [RS:1;4464c5b832df:44211 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C44211%2C1732423704184, suffix=, logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184, archiveDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs, maxLogs=32 2024-11-24T04:48:26,066 DEBUG [RS:0;4464c5b832df:42899 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,42899,1732423704094/4464c5b832df%2C42899%2C1732423704094.1732423706054, exclude list is [], retry=0 2024-11-24T04:48:26,066 DEBUG [RS:1;4464c5b832df:44211 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184/4464c5b832df%2C44211%2C1732423704184.1732423706054, exclude list is [], retry=0 2024-11-24T04:48:26,066 DEBUG [RS:2;4464c5b832df:33335 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223/4464c5b832df%2C33335%2C1732423704223.1732423706054, exclude list is [], retry=0 2024-11-24T04:48:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:26,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:26,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:26,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:26,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:26,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:26,105 INFO [RS:1;4464c5b832df:44211 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184/4464c5b832df%2C44211%2C1732423704184.1732423706054 2024-11-24T04:48:26,106 DEBUG [RS:1;4464c5b832df:44211 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:48:26,115 INFO [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223/4464c5b832df%2C33335%2C1732423704223.1732423706054 2024-11-24T04:48:26,115 DEBUG [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:26,117 INFO [RS:0;4464c5b832df:42899 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,42899,1732423704094/4464c5b832df%2C42899%2C1732423704094.1732423706054 2024-11-24T04:48:26,118 DEBUG [RS:0;4464c5b832df:42899 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:26,160 DEBUG [4464c5b832df:38057 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T04:48:26,170 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(204): Hosts are {4464c5b832df=0} racks are {/default-rack=0} 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T04:48:26,176 INFO [4464c5b832df:38057 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:48:26,176 INFO [4464c5b832df:38057 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:48:26,176 INFO [4464c5b832df:38057 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T04:48:26,176 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:48:26,182 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4464c5b832df,33335,1732423704223 2024-11-24T04:48:26,187 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4464c5b832df,33335,1732423704223, state=OPENING 2024-11-24T04:48:26,242 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T04:48:26,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:26,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:26,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:26,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:48:26,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,258 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:48:26,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4464c5b832df,33335,1732423704223}] 2024-11-24T04:48:26,443 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:48:26,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:48:26,456 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T04:48:26,456 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:26,457 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T04:48:26,460 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C33335%2C1732423704223.meta, suffix=.meta, logDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223, archiveDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs, maxLogs=32 2024-11-24T04:48:26,476 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223/4464c5b832df%2C33335%2C1732423704223.meta.1732423706463.meta, exclude list is [], retry=0 2024-11-24T04:48:26,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:26,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:26,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:26,507 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,33335,1732423704223/4464c5b832df%2C33335%2C1732423704223.meta.1732423706463.meta 2024-11-24T04:48:26,507 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:26,507 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:26,509 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T04:48:26,511 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T04:48:26,515 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T04:48:26,518 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T04:48:26,519 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:26,519 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T04:48:26,519 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T04:48:26,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:48:26,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:48:26,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:26,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:26,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:48:26,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:48:26,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:26,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:26,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:48:26,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:48:26,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:26,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:26,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:48:26,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:48:26,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:26,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:48:26,534 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:48:26,536 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740 2024-11-24T04:48:26,538 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740 2024-11-24T04:48:26,542 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:48:26,542 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:48:26,543 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:48:26,546 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:48:26,548 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63471879, jitterRate=-0.05419529974460602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:48:26,548 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T04:48:26,549 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732423706520Writing region info on filesystem at 1732423706520Initializing all the Stores at 1732423706522 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423706522Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423706522Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423706522Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423706522Cleaning up temporary data from old regions at 1732423706542 (+20 ms)Running coprocessor post-open hooks at 1732423706548 (+6 ms)Region opened successfully at 1732423706549 (+1 ms) 2024-11-24T04:48:26,555 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732423706434 2024-11-24T04:48:26,565 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T04:48:26,566 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T04:48:26,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4464c5b832df,33335,1732423704223 2024-11-24T04:48:26,570 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4464c5b832df,33335,1732423704223, state=OPEN 2024-11-24T04:48:26,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:48:26,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:48:26,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:48:26,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:48:26,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:48:26,624 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4464c5b832df,33335,1732423704223 2024-11-24T04:48:26,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T04:48:26,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4464c5b832df,33335,1732423704223 in 362 msec 2024-11-24T04:48:26,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T04:48:26,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 891 msec 2024-11-24T04:48:26,640 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:48:26,640 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T04:48:26,658 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:48:26,660 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4464c5b832df,33335,1732423704223, seqNum=-1] 2024-11-24T04:48:26,677 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:48:26,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49517, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:48:26,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2300 sec 2024-11-24T04:48:26,700 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732423706700, completionTime=-1 2024-11-24T04:48:26,702 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T04:48:26,702 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T04:48:26,750 INFO [master/4464c5b832df:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T04:48:26,750 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732423766750 2024-11-24T04:48:26,750 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732423826750 2024-11-24T04:48:26,751 INFO [master/4464c5b832df:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 48 msec 2024-11-24T04:48:26,756 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T04:48:26,767 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,38057,1732423703360-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,767 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,38057,1732423703360-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,767 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,38057,1732423703360-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,769 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4464c5b832df:38057, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,770 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,770 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,778 DEBUG [master/4464c5b832df:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T04:48:26,801 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.320sec 2024-11-24T04:48:26,803 INFO [master/4464c5b832df:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T04:48:26,804 INFO [master/4464c5b832df:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T04:48:26,804 INFO [master/4464c5b832df:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T04:48:26,805 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T04:48:26,805 INFO [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T04:48:26,806 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,38057,1732423703360-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:48:26,806 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,38057,1732423703360-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T04:48:26,811 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T04:48:26,812 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T04:48:26,812 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,38057,1732423703360-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:26,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a02bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:48:26,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4464c5b832df,38057,-1 for getting cluster id 2024-11-24T04:48:26,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:48:26,881 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '894908be-bd12-4c70-96f1-bb976b7347b7' 2024-11-24T04:48:26,883 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:48:26,883 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "894908be-bd12-4c70-96f1-bb976b7347b7" 2024-11-24T04:48:26,884 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1198a30f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:48:26,884 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4464c5b832df,38057,-1] 2024-11-24T04:48:26,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:48:26,889 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:48:26,890 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:48:26,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c1cca64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:48:26,893 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:48:26,899 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4464c5b832df,33335,1732423704223, seqNum=-1] 2024-11-24T04:48:26,900 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:48:26,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:48:26,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4464c5b832df,38057,1732423703360 2024-11-24T04:48:26,919 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43985/hbase 2024-11-24T04:48:26,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=586, ProcessCount=11, AvailableMemoryMB=10675 2024-11-24T04:48:26,953 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:26,957 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:26,958 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:26,964 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39787095, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-39787095, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:26,985 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39787095/hregion-39787095.1732423706966, exclude list is [], retry=0 2024-11-24T04:48:26,990 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:26,990 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:26,991 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:27,019 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39787095/hregion-39787095.1732423706966 2024-11-24T04:48:27,020 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:27,020 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c4a5f83c33415b05ff25c25bd1451a5e, NAME => 'testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:27,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:48:27,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:48:27,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:48:27,036 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:27,038 INFO [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,041 INFO [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a5f83c33415b05ff25c25bd1451a5e columnFamilyName a 2024-11-24T04:48:27,041 DEBUG [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:27,042 INFO [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] regionserver.HStore(327): Store=c4a5f83c33415b05ff25c25bd1451a5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:27,042 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,043 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,044 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,044 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,047 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,050 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:27,051 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4a5f83c33415b05ff25c25bd1451a5e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71981062, jitterRate=0.07260140776634216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:27,051 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4a5f83c33415b05ff25c25bd1451a5e: Writing region info on filesystem at 1732423707036Initializing all the Stores at 1732423707037 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423707037Cleaning up temporary data from old regions at 1732423707044 (+7 ms)Region opened successfully at 1732423707051 (+7 ms) 2024-11-24T04:48:27,052 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c4a5f83c33415b05ff25c25bd1451a5e, disabling compactions & flushes 2024-11-24T04:48:27,052 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e. 2024-11-24T04:48:27,052 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e. 2024-11-24T04:48:27,052 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e. after waiting 0 ms 2024-11-24T04:48:27,052 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e. 2024-11-24T04:48:27,052 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e. 2024-11-24T04:48:27,052 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c4a5f83c33415b05ff25c25bd1451a5e: Waiting for close lock at 1732423707052Disabling compacts and flushes for region at 1732423707052Disabling writes for close at 1732423707052Writing region close event to WAL at 1732423707052Closed at 1732423707052 2024-11-24T04:48:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741837_1013 (size=175) 2024-11-24T04:48:27,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741837_1013 (size=175) 2024-11-24T04:48:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741837_1013 (size=175) 2024-11-24T04:48:27,063 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:27,063 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39787095:(num 1732423706966) 2024-11-24T04:48:27,066 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:48:27,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741839_1015 (size=469) 2024-11-24T04:48:27,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741839_1015 (size=469) 2024-11-24T04:48:27,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741839_1015 (size=469) 2024-11-24T04:48:27,110 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:48:27,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741840_1016 (size=322) 2024-11-24T04:48:27,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741840_1016 (size=322) 2024-11-24T04:48:27,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741840_1016 (size=322) 2024-11-24T04:48:27,173 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1, size=469 (469bytes) 2024-11-24T04:48:27,174 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T04:48:27,175 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T04:48:27,175 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1 2024-11-24T04:48:27,181 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1 after 4ms 2024-11-24T04:48:27,211 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:27,212 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1 took 40ms 2024-11-24T04:48:27,217 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1 so closing down 2024-11-24T04:48:27,217 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:27,220 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T04:48:27,244 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T04:48:27,245 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:27,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741841_1017 (size=469) 2024-11-24T04:48:27,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741841_1017 (size=469) 2024-11-24T04:48:27,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741841_1017 (size=469) 2024-11-24T04:48:27,256 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:27,259 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002 2024-11-24T04:48:27,263 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 46 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-24T04:48:27,263 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1, journal: Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1, size=469 (469bytes) at 1732423707174Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1 so closing down at 1732423707217 (+43 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000001-wal-1.temp at 1732423707244 (+27 ms)3 split writer threads finished at 1732423707245 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732423707256 (+11 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002 at 1732423707259 (+3 ms)Processed 2 edits across 1 Regions in 46 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732423707263 (+4 ms) 2024-11-24T04:48:27,277 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2, size=322 (322bytes) 2024-11-24T04:48:27,277 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2 2024-11-24T04:48:27,278 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2 after 0ms 2024-11-24T04:48:27,304 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:27,304 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2 took 27ms 2024-11-24T04:48:27,307 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2 so closing down 2024-11-24T04:48:27,307 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:27,309 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T04:48:27,334 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T04:48:27,334 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:27,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741842_1018 (size=322) 2024-11-24T04:48:27,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741842_1018 (size=322) 2024-11-24T04:48:27,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741842_1018 (size=322) 2024-11-24T04:48:27,343 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:27,373 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:27,376 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-24T04:48:27,378 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-24T04:48:27,378 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2, journal: Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2, size=322 (322bytes) at 1732423707277Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2 so closing down at 1732423707307 (+30 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002-wal-2.temp at 1732423707334 (+27 ms)3 split writer threads finished at 1732423707334Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732423707343 (+9 ms)Processed 1 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732423707378 (+35 ms) 2024-11-24T04:48:27,378 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:27,380 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:27,393 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal.1732423707382, exclude list is [], retry=0 2024-11-24T04:48:27,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:27,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:27,398 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:27,421 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal.1732423707382 2024-11-24T04:48:27,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:27,422 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c4a5f83c33415b05ff25c25bd1451a5e, NAME => 'testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:27,422 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:27,422 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,422 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,425 INFO [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,426 INFO [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4a5f83c33415b05ff25c25bd1451a5e columnFamilyName a 2024-11-24T04:48:27,426 DEBUG [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:27,427 INFO [StoreOpener-c4a5f83c33415b05ff25c25bd1451a5e-1 {}] regionserver.HStore(327): Store=c4a5f83c33415b05ff25c25bd1451a5e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:27,427 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,428 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,431 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,432 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002 2024-11-24T04:48:27,457 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:27,462 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002 2024-11-24T04:48:27,465 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c4a5f83c33415b05ff25c25bd1451a5e 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T04:48:27,514 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/.tmp/a/85f1db3138074351987499ea2a4a788e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732423707063/Put/seqid=0 2024-11-24T04:48:27,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:48:27,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:48:27,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:48:27,527 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/.tmp/a/85f1db3138074351987499ea2a4a788e 2024-11-24T04:48:27,564 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/.tmp/a/85f1db3138074351987499ea2a4a788e as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/a/85f1db3138074351987499ea2a4a788e 2024-11-24T04:48:27,573 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/a/85f1db3138074351987499ea2a4a788e, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T04:48:27,579 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c4a5f83c33415b05ff25c25bd1451a5e in 112ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T04:48:27,581 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/0000000000000000002 2024-11-24T04:48:27,582 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,582 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,585 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c4a5f83c33415b05ff25c25bd1451a5e 2024-11-24T04:48:27,588 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/c4a5f83c33415b05ff25c25bd1451a5e/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T04:48:27,590 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c4a5f83c33415b05ff25c25bd1451a5e; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60311369, jitterRate=-0.10129056870937347}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:27,591 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c4a5f83c33415b05ff25c25bd1451a5e: Writing region info on filesystem at 1732423707422Initializing all the Stores at 1732423707424 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423707424Obtaining lock to block concurrent updates at 1732423707465 (+41 ms)Preparing flush snapshotting stores in c4a5f83c33415b05ff25c25bd1451a5e at 1732423707465Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732423707468 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732423706954.c4a5f83c33415b05ff25c25bd1451a5e. at 1732423707468Flushing c4a5f83c33415b05ff25c25bd1451a5e/a: creating writer at 1732423707469 (+1 ms)Flushing c4a5f83c33415b05ff25c25bd1451a5e/a: appending metadata at 1732423707503 (+34 ms)Flushing c4a5f83c33415b05ff25c25bd1451a5e/a: closing flushed file at 1732423707505 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15e073fd: reopening flushed file at 1732423707563 (+58 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c4a5f83c33415b05ff25c25bd1451a5e in 112ms, sequenceid=2, compaction requested=false; wal=null at 1732423707579 (+16 ms)Cleaning up temporary data from old regions at 1732423707582 (+3 ms)Region opened successfully at 1732423707590 (+8 ms) 2024-11-24T04:48:27,613 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 362) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42122 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:33162 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37618 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42268 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43985/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37752 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:33184 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=586 (was 586), ProcessCount=11 (was 11), AvailableMemoryMB=10663 (was 10675) 2024-11-24T04:48:27,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=586, ProcessCount=11, AvailableMemoryMB=10662 2024-11-24T04:48:27,641 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:27,644 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:27,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:27,648 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53683512, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-53683512, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:27,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53683512/hregion-53683512.1732423707649, exclude list is [], retry=0 2024-11-24T04:48:27,663 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:27,663 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:27,663 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:27,690 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53683512/hregion-53683512.1732423707649 2024-11-24T04:48:27,691 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:48:27,691 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6d7a2ad85129012bb12243b10096a106, NAME => 'testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:27,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:48:27,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:48:27,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:48:27,705 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:27,707 INFO [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,709 INFO [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7a2ad85129012bb12243b10096a106 columnFamilyName a 2024-11-24T04:48:27,709 DEBUG [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:27,710 INFO [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] regionserver.HStore(327): Store=6d7a2ad85129012bb12243b10096a106/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:27,710 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,712 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,713 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,715 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:27,718 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:27,718 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6d7a2ad85129012bb12243b10096a106; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71980425, jitterRate=0.07259191572666168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:27,719 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6d7a2ad85129012bb12243b10096a106: Writing region info on filesystem at 1732423707705Initializing all the Stores at 1732423707706 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423707706Cleaning up temporary data from old regions at 1732423707713 (+7 ms)Region opened successfully at 1732423707719 (+6 ms) 2024-11-24T04:48:27,719 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6d7a2ad85129012bb12243b10096a106, disabling compactions & flushes 2024-11-24T04:48:27,719 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106. 2024-11-24T04:48:27,719 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106. 2024-11-24T04:48:27,719 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106. after waiting 0 ms 2024-11-24T04:48:27,719 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106. 2024-11-24T04:48:27,719 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106. 2024-11-24T04:48:27,719 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6d7a2ad85129012bb12243b10096a106: Waiting for close lock at 1732423707719Disabling compacts and flushes for region at 1732423707719Disabling writes for close at 1732423707719Writing region close event to WAL at 1732423707719Closed at 1732423707719 2024-11-24T04:48:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741845_1021 (size=175) 2024-11-24T04:48:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741845_1021 (size=175) 2024-11-24T04:48:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741845_1021 (size=175) 2024-11-24T04:48:27,726 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:27,726 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53683512:(num 1732423707649) 2024-11-24T04:48:27,727 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:48:27,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741847_1023 (size=469) 2024-11-24T04:48:27,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741847_1023 (size=469) 2024-11-24T04:48:27,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741847_1023 (size=469) 2024-11-24T04:48:27,763 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:48:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741848_1024 (size=322) 2024-11-24T04:48:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741848_1024 (size=322) 2024-11-24T04:48:27,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741848_1024 (size=322) 2024-11-24T04:48:27,815 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2, size=322 (322bytes) 2024-11-24T04:48:27,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2 2024-11-24T04:48:27,816 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2 after 0ms 2024-11-24T04:48:27,841 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:27,841 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2 took 27ms 2024-11-24T04:48:27,844 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2 so closing down 2024-11-24T04:48:27,844 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:27,846 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T04:48:27,872 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T04:48:27,873 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:27,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741849_1025 (size=322) 2024-11-24T04:48:27,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741849_1025 (size=322) 2024-11-24T04:48:27,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741849_1025 (size=322) 2024-11-24T04:48:27,882 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:27,884 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 2024-11-24T04:48:27,885 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-24T04:48:27,885 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2, journal: Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2, size=322 (322bytes) at 1732423707815Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2 so closing down at 1732423707844 (+29 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002-wal-2.temp at 1732423707872 (+28 ms)3 split writer threads finished at 1732423707873 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732423707882 (+9 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 at 1732423707884 (+2 ms)Processed 1 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732423707885 (+1 ms) 2024-11-24T04:48:27,901 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1, size=469 (469bytes) 2024-11-24T04:48:27,901 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1 2024-11-24T04:48:27,902 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1 after 1ms 2024-11-24T04:48:27,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:27,928 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1 took 27ms 2024-11-24T04:48:27,932 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1 so closing down 2024-11-24T04:48:27,932 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:27,934 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T04:48:27,962 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T04:48:27,963 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:27,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741850_1026 (size=469) 2024-11-24T04:48:27,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741850_1026 (size=469) 2024-11-24T04:48:27,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741850_1026 (size=469) 2024-11-24T04:48:27,972 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:27,999 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:28,002 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002, length=322 2024-11-24T04:48:28,004 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 2024-11-24T04:48:28,004 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-24T04:48:28,005 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1, journal: Splitting hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1, size=469 (469bytes) at 1732423707901Finishing writing output for hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1 so closing down at 1732423707932 (+31 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000001-wal-1.temp at 1732423707962 (+30 ms)3 split writer threads finished at 1732423707963 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732423707972 (+9 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 at 1732423708004 (+32 ms)Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732423708005 (+1 ms) 2024-11-24T04:48:28,005 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:28,007 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:28,021 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal.1732423708008, exclude list is [], retry=0 2024-11-24T04:48:28,024 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:28,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:28,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:28,049 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal.1732423708008 2024-11-24T04:48:28,050 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:28,050 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d7a2ad85129012bb12243b10096a106, NAME => 'testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:28,050 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:28,050 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,050 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,052 INFO [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,054 INFO [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7a2ad85129012bb12243b10096a106 columnFamilyName a 2024-11-24T04:48:28,054 DEBUG [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:28,055 INFO [StoreOpener-6d7a2ad85129012bb12243b10096a106-1 {}] regionserver.HStore(327): Store=6d7a2ad85129012bb12243b10096a106/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:28,055 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,056 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,058 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,059 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 2024-11-24T04:48:28,086 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:28,088 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 2024-11-24T04:48:28,088 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6d7a2ad85129012bb12243b10096a106 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T04:48:28,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/.tmp/a/df3c6fd044b04cac92fa7e0722ea19c2 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732423707726/Put/seqid=0 2024-11-24T04:48:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:48:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:48:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:48:28,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/.tmp/a/df3c6fd044b04cac92fa7e0722ea19c2 2024-11-24T04:48:28,128 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/.tmp/a/df3c6fd044b04cac92fa7e0722ea19c2 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/a/df3c6fd044b04cac92fa7e0722ea19c2 2024-11-24T04:48:28,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/a/df3c6fd044b04cac92fa7e0722ea19c2, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T04:48:28,139 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6d7a2ad85129012bb12243b10096a106 in 51ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T04:48:28,140 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/0000000000000000002 2024-11-24T04:48:28,141 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,141 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,144 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6d7a2ad85129012bb12243b10096a106 2024-11-24T04:48:28,147 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/6d7a2ad85129012bb12243b10096a106/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T04:48:28,150 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6d7a2ad85129012bb12243b10096a106; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60785477, jitterRate=-0.09422580897808075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:28,150 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6d7a2ad85129012bb12243b10096a106: Writing region info on filesystem at 1732423708050Initializing all the Stores at 1732423708052 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423708052Obtaining lock to block concurrent updates at 1732423708088 (+36 ms)Preparing flush snapshotting stores in 6d7a2ad85129012bb12243b10096a106 at 1732423708088Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732423708089 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732423707642.6d7a2ad85129012bb12243b10096a106. at 1732423708089Flushing 6d7a2ad85129012bb12243b10096a106/a: creating writer at 1732423708089Flushing 6d7a2ad85129012bb12243b10096a106/a: appending metadata at 1732423708103 (+14 ms)Flushing 6d7a2ad85129012bb12243b10096a106/a: closing flushed file at 1732423708103Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1137b54a: reopening flushed file at 1732423708126 (+23 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6d7a2ad85129012bb12243b10096a106 in 51ms, sequenceid=2, compaction requested=false; wal=null at 1732423708139 (+13 ms)Cleaning up temporary data from old regions at 1732423708141 (+2 ms)Region opened successfully at 1732423708150 (+9 ms) 2024-11-24T04:48:28,175 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42122 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42338 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:33260 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37618 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37806 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=586 (was 586), ProcessCount=11 (was 11), AvailableMemoryMB=10646 (was 10662) 2024-11-24T04:48:28,208 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=586, ProcessCount=11, AvailableMemoryMB=10644 2024-11-24T04:48:28,232 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:28,235 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:28,237 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:28,247 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-04251526, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-04251526, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:28,269 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-04251526/hregion-04251526.1732423708252, exclude list is [], retry=0 2024-11-24T04:48:28,284 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:28,297 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:28,310 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:28,363 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-04251526/hregion-04251526.1732423708252 2024-11-24T04:48:28,364 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:28,365 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => acd8b601ff7944915fc7c7570699f169, NAME => 'testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:28,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:48:28,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:48:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:48:28,400 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:28,405 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,407 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8b601ff7944915fc7c7570699f169 columnFamilyName a 2024-11-24T04:48:28,408 DEBUG [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:28,408 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(327): Store=acd8b601ff7944915fc7c7570699f169/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:28,409 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,414 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8b601ff7944915fc7c7570699f169 columnFamilyName b 2024-11-24T04:48:28,414 DEBUG [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:28,415 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(327): Store=acd8b601ff7944915fc7c7570699f169/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:28,415 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,417 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8b601ff7944915fc7c7570699f169 columnFamilyName c 2024-11-24T04:48:28,417 DEBUG [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:28,418 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(327): Store=acd8b601ff7944915fc7c7570699f169/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:28,418 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,422 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,422 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,423 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:28,424 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:28,427 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:28,427 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acd8b601ff7944915fc7c7570699f169; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72720397, jitterRate=0.08361835777759552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:28,428 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acd8b601ff7944915fc7c7570699f169: Writing region info on filesystem at 1732423708400Initializing all the Stores at 1732423708404 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423708404Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423708405 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423708405Cleaning up temporary data from old regions at 1732423708422 (+17 ms)Region opened successfully at 1732423708428 (+6 ms) 2024-11-24T04:48:28,428 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acd8b601ff7944915fc7c7570699f169, disabling compactions & flushes 2024-11-24T04:48:28,428 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:28,428 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:28,428 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. after waiting 0 ms 2024-11-24T04:48:28,428 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:28,429 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:28,429 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acd8b601ff7944915fc7c7570699f169: Waiting for close lock at 1732423708428Disabling compacts and flushes for region at 1732423708428Disabling writes for close at 1732423708428Writing region close event to WAL at 1732423708429 (+1 ms)Closed at 1732423708429 2024-11-24T04:48:28,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741853_1029 (size=175) 2024-11-24T04:48:28,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741853_1029 (size=175) 2024-11-24T04:48:28,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741853_1029 (size=175) 2024-11-24T04:48:28,438 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:28,439 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-04251526:(num 1732423708252) 2024-11-24T04:48:28,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:28,441 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:28,461 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442, exclude list is [], retry=0 2024-11-24T04:48:28,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:28,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:28,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:28,511 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 2024-11-24T04:48:28,512 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:28,815 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442, size=0 (0bytes) 2024-11-24T04:48:28,815 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 might be still open, length is 0 2024-11-24T04:48:28,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 2024-11-24T04:48:28,816 WARN [IPC Server handler 2 on default port 43985 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T04:48:28,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 after 2ms 2024-11-24T04:48:31,531 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37832 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43379:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37832 dst: /127.0.0.1:43379 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43379 remote=/127.0.0.1:37832]. Total timeout mills is 60000, 57231 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:31,532 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42372 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42372 dst: /127.0.0.1:35959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:31,533 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:33278 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45441:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33278 dst: /127.0.0.1:45441 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:31,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741855_1032 (size=497775) 2024-11-24T04:48:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741855_1032 (size=497775) 2024-11-24T04:48:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741855_1032 (size=497775) 2024-11-24T04:48:32,035 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:48:32,122 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T04:48:32,818 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 after 4003ms 2024-11-24T04:48:32,849 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:32,851 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 took 4037ms 2024-11-24T04:48:32,858 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732423708442.temp 2024-11-24T04:48:32,915 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000000001-wal.1732423708442.temp 2024-11-24T04:48:32,989 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442; continuing. 2024-11-24T04:48:32,989 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 so closing down 2024-11-24T04:48:32,989 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:32,989 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:33,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741856_1033 (size=497783) 2024-11-24T04:48:33,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741856_1033 (size=497783) 2024-11-24T04:48:33,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741856_1033 (size=497783) 2024-11-24T04:48:33,007 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000000001-wal.1732423708442.temp (wrote 3002 edits, skipped 0 edits in 69 ms) 2024-11-24T04:48:33,010 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000000001-wal.1732423708442.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000003002 2024-11-24T04:48:33,010 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 159 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:48:33,010 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442, journal: Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442, size=0 (0bytes) at 1732423708815Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000000001-wal.1732423708442.temp at 1732423712915 (+4100 ms)Split 1024 edits, skipped 0 edits. at 1732423712922 (+7 ms)Split 2048 edits, skipped 0 edits. at 1732423712965 (+43 ms)Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 so closing down at 1732423712989 (+24 ms)3 split writer threads finished at 1732423712989Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000000001-wal.1732423708442.temp (wrote 3002 edits, skipped 0 edits in 69 ms) at 1732423713007 (+18 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000000001-wal.1732423708442.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000003002 at 1732423713010 (+3 ms)Processed 3002 edits across 1 Regions in 159 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442, size=0, length=0, corrupted=false, cancelled=false at 1732423713010 2024-11-24T04:48:33,015 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423708442 2024-11-24T04:48:33,019 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000003002 2024-11-24T04:48:33,019 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:33,022 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:33,038 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423713023, exclude list is [], retry=0 2024-11-24T04:48:33,043 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:33,044 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:33,044 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:33,072 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423713023 2024-11-24T04:48:33,073 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:33,073 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:33,076 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:33,078 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8b601ff7944915fc7c7570699f169 columnFamilyName a 2024-11-24T04:48:33,078 DEBUG [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:33,079 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(327): Store=acd8b601ff7944915fc7c7570699f169/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:33,079 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:33,080 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8b601ff7944915fc7c7570699f169 columnFamilyName b 2024-11-24T04:48:33,080 DEBUG [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:33,081 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(327): Store=acd8b601ff7944915fc7c7570699f169/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:33,081 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:33,082 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acd8b601ff7944915fc7c7570699f169 columnFamilyName c 2024-11-24T04:48:33,082 DEBUG [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:33,083 INFO [StoreOpener-acd8b601ff7944915fc7c7570699f169-1 {}] regionserver.HStore(327): Store=acd8b601ff7944915fc7c7570699f169/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:33,084 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:33,085 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:33,090 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:33,091 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000003002 2024-11-24T04:48:33,136 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:33,196 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:48:33,702 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T04:48:33,703 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:33,715 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:48:33,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T04:48:33,716 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T04:48:33,716 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:33,717 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T04:48:33,717 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:33,734 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acd8b601ff7944915fc7c7570699f169 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:48:33,795 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/a/d9433201e1b04093ae5389d194970d1a is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732423708525/Put/seqid=0 2024-11-24T04:48:33,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:48:33,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:48:33,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:48:33,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/a/d9433201e1b04093ae5389d194970d1a 2024-11-24T04:48:33,839 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/a/d9433201e1b04093ae5389d194970d1a as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/a/d9433201e1b04093ae5389d194970d1a 2024-11-24T04:48:33,854 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/a/d9433201e1b04093ae5389d194970d1a, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T04:48:33,855 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for acd8b601ff7944915fc7c7570699f169 in 122ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T04:48:33,883 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:48:33,883 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acd8b601ff7944915fc7c7570699f169 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:48:33,901 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/a/151466c508dd4f858a240559cab4d8ce is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732423708587/Put/seqid=0 2024-11-24T04:48:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:48:33,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:48:33,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:48:33,941 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/a/151466c508dd4f858a240559cab4d8ce 2024-11-24T04:48:33,978 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/b/be2006c4edb44348a868e03be3a0a4c3 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732423708623/Put/seqid=0 2024-11-24T04:48:34,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:48:34,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:48:34,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:48:34,025 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/b/be2006c4edb44348a868e03be3a0a4c3 2024-11-24T04:48:34,033 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/a/151466c508dd4f858a240559cab4d8ce as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/a/151466c508dd4f858a240559cab4d8ce 2024-11-24T04:48:34,049 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/a/151466c508dd4f858a240559cab4d8ce, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T04:48:34,050 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/b/be2006c4edb44348a868e03be3a0a4c3 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/b/be2006c4edb44348a868e03be3a0a4c3 2024-11-24T04:48:34,059 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/b/be2006c4edb44348a868e03be3a0a4c3, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T04:48:34,060 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for acd8b601ff7944915fc7c7570699f169 in 177ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T04:48:34,084 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:48:34,084 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acd8b601ff7944915fc7c7570699f169 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:48:34,099 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/b/203ef7dd01fb4c488af018e1946830f7 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732423708658/Put/seqid=0 2024-11-24T04:48:34,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:48:34,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:48:34,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:48:34,162 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/b/203ef7dd01fb4c488af018e1946830f7 2024-11-24T04:48:34,191 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/c/8006d39e928144758d0338740a3f7d6d is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732423708703/Put/seqid=0 2024-11-24T04:48:34,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:48:34,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:48:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:48:34,209 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/c/8006d39e928144758d0338740a3f7d6d 2024-11-24T04:48:34,219 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/b/203ef7dd01fb4c488af018e1946830f7 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/b/203ef7dd01fb4c488af018e1946830f7 2024-11-24T04:48:34,228 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/b/203ef7dd01fb4c488af018e1946830f7, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T04:48:34,230 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/c/8006d39e928144758d0338740a3f7d6d as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/c/8006d39e928144758d0338740a3f7d6d 2024-11-24T04:48:34,238 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/c/8006d39e928144758d0338740a3f7d6d, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T04:48:34,239 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for acd8b601ff7944915fc7c7570699f169 in 155ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T04:48:34,255 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732423708759/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:34,260 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000003002 2024-11-24T04:48:34,261 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:48:34,261 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing acd8b601ff7944915fc7c7570699f169 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T04:48:34,278 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/c/9819b541f1eb4369b62b0b645f0da6ca is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732423708713/Put/seqid=0 2024-11-24T04:48:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:48:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:48:34,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:48:34,295 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/c/9819b541f1eb4369b62b0b645f0da6ca 2024-11-24T04:48:34,302 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9819b541f1eb4369b62b0b645f0da6ca 2024-11-24T04:48:34,303 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/.tmp/c/9819b541f1eb4369b62b0b645f0da6ca as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/c/9819b541f1eb4369b62b0b645f0da6ca 2024-11-24T04:48:34,311 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9819b541f1eb4369b62b0b645f0da6ca 2024-11-24T04:48:34,311 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/c/9819b541f1eb4369b62b0b645f0da6ca, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T04:48:34,312 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for acd8b601ff7944915fc7c7570699f169 in 50ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T04:48:34,313 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/0000000000000003002 2024-11-24T04:48:34,315 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:34,315 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:34,316 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T04:48:34,318 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for acd8b601ff7944915fc7c7570699f169 2024-11-24T04:48:34,322 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenIntoWAL/acd8b601ff7944915fc7c7570699f169/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T04:48:34,324 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened acd8b601ff7944915fc7c7570699f169; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62522802, jitterRate=-0.06833764910697937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T04:48:34,324 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for acd8b601ff7944915fc7c7570699f169: Writing region info on filesystem at 1732423713073Initializing all the Stores at 1732423713074 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423713074Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423713076 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423713076Cleaning up temporary data from old regions at 1732423714315 (+1239 ms)Region opened successfully at 1732423714324 (+9 ms) 2024-11-24T04:48:34,447 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing acd8b601ff7944915fc7c7570699f169, disabling compactions & flushes 2024-11-24T04:48:34,447 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:34,447 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:34,447 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. after waiting 0 ms 2024-11-24T04:48:34,447 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:34,461 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423708233.acd8b601ff7944915fc7c7570699f169. 2024-11-24T04:48:34,461 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for acd8b601ff7944915fc7c7570699f169: Waiting for close lock at 1732423714447Disabling compacts and flushes for region at 1732423714447Disabling writes for close at 1732423714447Writing region close event to WAL at 1732423714461 (+14 ms)Closed at 1732423714461 2024-11-24T04:48:34,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741857_1034 (size=175) 2024-11-24T04:48:34,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741857_1034 (size=175) 2024-11-24T04:48:34,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741857_1034 (size=175) 2024-11-24T04:48:34,483 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:34,484 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423713023) 2024-11-24T04:48:34,508 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=398 (was 384) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:40807 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40807 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1959481366_22 at /127.0.0.1:51682 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:36887 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36887 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1959481366_22 at /127.0.0.1:35668 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1959481366_22 at /127.0.0.1:42514 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1959481366_22 at /127.0.0.1:35624 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:43985 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43985 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 586), ProcessCount=11 (was 11), AvailableMemoryMB=10531 (was 10644) 2024-11-24T04:48:34,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=398, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=579, ProcessCount=11, AvailableMemoryMB=10525 2024-11-24T04:48:34,564 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:34,569 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:34,570 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:34,579 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22778230, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-22778230, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:34,599 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22778230/hregion-22778230.1732423714580, exclude list is [], retry=0 2024-11-24T04:48:34,605 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:34,615 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:34,617 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:34,688 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22778230/hregion-22778230.1732423714580 2024-11-24T04:48:34,689 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:34,690 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dbc148e176263d7ac2c3b7826938fbc2, NAME => 'test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:34,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:48:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:48:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:48:35,138 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:35,140 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,143 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc148e176263d7ac2c3b7826938fbc2 columnFamilyName a 2024-11-24T04:48:35,143 DEBUG [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:35,144 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(327): Store=dbc148e176263d7ac2c3b7826938fbc2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:35,144 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,151 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc148e176263d7ac2c3b7826938fbc2 columnFamilyName b 2024-11-24T04:48:35,151 DEBUG [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:35,152 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(327): Store=dbc148e176263d7ac2c3b7826938fbc2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:35,154 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,159 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc148e176263d7ac2c3b7826938fbc2 columnFamilyName c 2024-11-24T04:48:35,159 DEBUG [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:35,160 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(327): Store=dbc148e176263d7ac2c3b7826938fbc2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:35,160 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,167 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,171 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,171 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,172 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:35,176 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:35,181 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:35,184 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbc148e176263d7ac2c3b7826938fbc2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61628025, jitterRate=-0.0816708654165268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbc148e176263d7ac2c3b7826938fbc2: Writing region info on filesystem at 1732423715138Initializing all the Stores at 1732423715140 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423715140Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423715140Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423715140Cleaning up temporary data from old regions at 1732423715171 (+31 ms)Region opened successfully at 1732423715185 (+14 ms) 2024-11-24T04:48:35,185 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbc148e176263d7ac2c3b7826938fbc2, disabling compactions & flushes 2024-11-24T04:48:35,185 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:35,186 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:35,186 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. after waiting 0 ms 2024-11-24T04:48:35,186 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:35,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:35,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbc148e176263d7ac2c3b7826938fbc2: Waiting for close lock at 1732423715185Disabling compacts and flushes for region at 1732423715185Disabling writes for close at 1732423715186 (+1 ms)Writing region close event to WAL at 1732423715186Closed at 1732423715186 2024-11-24T04:48:35,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741864_1041 (size=175) 2024-11-24T04:48:35,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741864_1041 (size=175) 2024-11-24T04:48:35,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741864_1041 (size=175) 2024-11-24T04:48:35,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:35,196 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22778230:(num 1732423714580) 2024-11-24T04:48:35,196 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:35,198 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:35,211 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198, exclude list is [], retry=0 2024-11-24T04:48:35,215 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:35,215 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:35,216 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:35,246 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 2024-11-24T04:48:35,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741866_1043 (size=371388) 2024-11-24T04:48:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741866_1043 (size=371388) 2024-11-24T04:48:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741866_1043 (size=371388) 2024-11-24T04:48:35,539 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198, size=362.7 K (371388bytes) 2024-11-24T04:48:35,539 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 2024-11-24T04:48:35,539 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 after 0ms 2024-11-24T04:48:35,579 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:35,580 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 took 41ms 2024-11-24T04:48:35,587 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732423715198.temp 2024-11-24T04:48:35,630 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 so closing down 2024-11-24T04:48:35,630 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:35,650 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000000001-wal.1732423715198.temp 2024-11-24T04:48:35,672 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:35,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741867_1044 (size=371388) 2024-11-24T04:48:35,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741867_1044 (size=371388) 2024-11-24T04:48:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741867_1044 (size=371388) 2024-11-24T04:48:35,719 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000000001-wal.1732423715198.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-24T04:48:35,726 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000000001-wal.1732423715198.temp to hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003000 2024-11-24T04:48:35,726 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 145 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-24T04:48:35,727 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198, journal: Splitting hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198, size=362.7 K (371388bytes) at 1732423715539Split 1024 edits, skipped 0 edits. at 1732423715596 (+57 ms)Split 2048 edits, skipped 0 edits. at 1732423715609 (+13 ms)Finishing writing output for hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 so closing down at 1732423715630 (+21 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000000001-wal.1732423715198.temp at 1732423715650 (+20 ms)3 split writer threads finished at 1732423715672 (+22 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000000001-wal.1732423715198.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1732423715719 (+47 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000000001-wal.1732423715198.temp to hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003000 at 1732423715726 (+7 ms)Processed 3000 edits across 1 Regions in 145 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732423715726 2024-11-24T04:48:35,729 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715198 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423715198 2024-11-24T04:48:35,732 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003000 2024-11-24T04:48:35,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:35,736 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:35,753 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737, exclude list is [], retry=0 2024-11-24T04:48:35,757 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:35,758 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:35,759 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:35,796 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 2024-11-24T04:48:35,797 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:48:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741868_1045 (size=371515) 2024-11-24T04:48:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741868_1045 (size=371515) 2024-11-24T04:48:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741868_1045 (size=371515) 2024-11-24T04:48:36,092 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737, size=362.8 K (371515bytes) 2024-11-24T04:48:36,092 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 2024-11-24T04:48:36,092 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 after 0ms 2024-11-24T04:48:36,135 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:36,136 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 took 45ms 2024-11-24T04:48:36,143 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732423715737.temp 2024-11-24T04:48:36,209 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 so closing down 2024-11-24T04:48:36,210 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:36,222 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003001-wal.1732423715737.temp 2024-11-24T04:48:36,239 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:36,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741869_1046 (size=371515) 2024-11-24T04:48:36,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741869_1046 (size=371515) 2024-11-24T04:48:36,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741869_1046 (size=371515) 2024-11-24T04:48:36,293 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003001-wal.1732423715737.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-11-24T04:48:36,295 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003001-wal.1732423715737.temp to hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000006000 2024-11-24T04:48:36,296 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 159 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-24T04:48:36,296 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737, journal: Splitting hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737, size=362.8 K (371515bytes) at 1732423716092Split 1024 edits, skipped 0 edits. at 1732423716153 (+61 ms)Split 2048 edits, skipped 0 edits. at 1732423716202 (+49 ms)Finishing writing output for hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 so closing down at 1732423716209 (+7 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003001-wal.1732423715737.temp at 1732423716223 (+14 ms)3 split writer threads finished at 1732423716239 (+16 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003001-wal.1732423715737.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1732423716293 (+54 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003001-wal.1732423715737.temp to hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000006000 at 1732423716296 (+3 ms)Processed 3000 edits across 1 Regions in 159 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732423716296 2024-11-24T04:48:36,298 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423715737 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423715737 2024-11-24T04:48:36,299 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000006000 2024-11-24T04:48:36,299 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:36,301 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/test2727-manual,16010,1732423714560, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:36,314 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423716301, exclude list is [], retry=0 2024-11-24T04:48:36,316 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:36,317 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:36,317 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:36,345 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732423714560/wal.1732423716301 2024-11-24T04:48:36,346 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:36,346 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dbc148e176263d7ac2c3b7826938fbc2, NAME => 'test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:36,346 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:36,346 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,346 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,348 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,349 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc148e176263d7ac2c3b7826938fbc2 columnFamilyName a 2024-11-24T04:48:36,349 DEBUG [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:36,350 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(327): Store=dbc148e176263d7ac2c3b7826938fbc2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:36,350 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,351 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc148e176263d7ac2c3b7826938fbc2 columnFamilyName b 2024-11-24T04:48:36,351 DEBUG [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:36,351 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(327): Store=dbc148e176263d7ac2c3b7826938fbc2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:36,351 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,352 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbc148e176263d7ac2c3b7826938fbc2 columnFamilyName c 2024-11-24T04:48:36,352 DEBUG [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:36,353 INFO [StoreOpener-dbc148e176263d7ac2c3b7826938fbc2-1 {}] regionserver.HStore(327): Store=dbc148e176263d7ac2c3b7826938fbc2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:36,353 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,354 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,358 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,359 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003000 2024-11-24T04:48:36,389 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:36,449 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003000 2024-11-24T04:48:36,455 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000006000 2024-11-24T04:48:36,500 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:36,532 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000006000 2024-11-24T04:48:36,532 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dbc148e176263d7ac2c3b7826938fbc2 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T04:48:36,560 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/a/7cdc7ebbbe374ca2985647f6ee910fef is 41, key is test2727/a:100/1732423715803/Put/seqid=0 2024-11-24T04:48:36,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:48:36,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:48:36,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:48:36,593 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/a/7cdc7ebbbe374ca2985647f6ee910fef 2024-11-24T04:48:36,639 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/b/d52f552471c0432fa430582308a35db4 is 41, key is test2727/b:100/1732423715915/Put/seqid=0 2024-11-24T04:48:36,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:48:36,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:48:36,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:48:36,677 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/b/d52f552471c0432fa430582308a35db4 2024-11-24T04:48:36,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/c/a77f1b14f9e1484f88cb4b4d0204e826 is 41, key is test2727/c:100/1732423715980/Put/seqid=0 2024-11-24T04:48:36,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:48:36,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:48:36,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:48:36,756 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/c/a77f1b14f9e1484f88cb4b4d0204e826 2024-11-24T04:48:36,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/a/7cdc7ebbbe374ca2985647f6ee910fef as hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/a/7cdc7ebbbe374ca2985647f6ee910fef 2024-11-24T04:48:36,784 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/a/7cdc7ebbbe374ca2985647f6ee910fef, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T04:48:36,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/b/d52f552471c0432fa430582308a35db4 as hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/b/d52f552471c0432fa430582308a35db4 2024-11-24T04:48:36,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/b/d52f552471c0432fa430582308a35db4, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T04:48:36,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/.tmp/c/a77f1b14f9e1484f88cb4b4d0204e826 as hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/c/a77f1b14f9e1484f88cb4b4d0204e826 2024-11-24T04:48:36,806 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/c/a77f1b14f9e1484f88cb4b4d0204e826, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T04:48:36,807 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for dbc148e176263d7ac2c3b7826938fbc2 in 274ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T04:48:36,809 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000003000 2024-11-24T04:48:36,809 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/0000000000000006000 2024-11-24T04:48:36,810 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,811 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,812 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:36,814 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dbc148e176263d7ac2c3b7826938fbc2 2024-11-24T04:48:36,818 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/test2727/dbc148e176263d7ac2c3b7826938fbc2/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T04:48:36,821 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dbc148e176263d7ac2c3b7826938fbc2; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72382396, jitterRate=0.07858175039291382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:36,822 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dbc148e176263d7ac2c3b7826938fbc2: Writing region info on filesystem at 1732423716346Initializing all the Stores at 1732423716348 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423716348Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423716348Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423716348Obtaining lock to block concurrent updates at 1732423716533 (+185 ms)Preparing flush snapshotting stores in dbc148e176263d7ac2c3b7826938fbc2 at 1732423716533Finished memstore snapshotting test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732423716533Flushing stores of test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. at 1732423716533Flushing dbc148e176263d7ac2c3b7826938fbc2/a: creating writer at 1732423716533Flushing dbc148e176263d7ac2c3b7826938fbc2/a: appending metadata at 1732423716558 (+25 ms)Flushing dbc148e176263d7ac2c3b7826938fbc2/a: closing flushed file at 1732423716558Flushing dbc148e176263d7ac2c3b7826938fbc2/b: creating writer at 1732423716604 (+46 ms)Flushing dbc148e176263d7ac2c3b7826938fbc2/b: appending metadata at 1732423716637 (+33 ms)Flushing dbc148e176263d7ac2c3b7826938fbc2/b: closing flushed file at 1732423716637Flushing dbc148e176263d7ac2c3b7826938fbc2/c: creating writer at 1732423716685 (+48 ms)Flushing dbc148e176263d7ac2c3b7826938fbc2/c: appending metadata at 1732423716715 (+30 ms)Flushing dbc148e176263d7ac2c3b7826938fbc2/c: closing flushed file at 1732423716715Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c772292: reopening flushed file at 1732423716771 (+56 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@332e1169: reopening flushed file at 1732423716784 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bd42cbb: reopening flushed file at 1732423716795 (+11 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for dbc148e176263d7ac2c3b7826938fbc2 in 274ms, sequenceid=6000, compaction requested=false; wal=null at 1732423716808 (+13 ms)Cleaning up temporary data from old regions at 1732423716811 (+3 ms)Region opened successfully at 1732423716821 (+10 ms) 2024-11-24T04:48:36,823 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T04:48:36,823 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dbc148e176263d7ac2c3b7826938fbc2, disabling compactions & flushes 2024-11-24T04:48:36,823 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:36,823 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:36,823 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. after waiting 0 ms 2024-11-24T04:48:36,823 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:36,835 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732423714566.dbc148e176263d7ac2c3b7826938fbc2. 2024-11-24T04:48:36,836 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dbc148e176263d7ac2c3b7826938fbc2: Waiting for close lock at 1732423716823Disabling compacts and flushes for region at 1732423716823Disabling writes for close at 1732423716823Writing region close event to WAL at 1732423716835 (+12 ms)Closed at 1732423716835 2024-11-24T04:48:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741870_1047 (size=175) 2024-11-24T04:48:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741870_1047 (size=175) 2024-11-24T04:48:36,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741870_1047 (size=175) 2024-11-24T04:48:36,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:36,844 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423716301) 2024-11-24T04:48:36,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399 (was 398) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:51760 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35754 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42514 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=588 (was 579) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10306 (was 10525) 2024-11-24T04:48:36,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=399, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=588, ProcessCount=11, AvailableMemoryMB=10306 2024-11-24T04:48:36,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:36,901 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:36,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732423716902 2024-11-24T04:48:36,945 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902 2024-11-24T04:48:36,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:36,964 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bf24f627d705b5969ef5403995b49a5b, NAME => 'testSequentialEditLogSeqNum,,1732423716896.bf24f627d705b5969ef5403995b49a5b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:36,964 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732423716896.bf24f627d705b5969ef5403995b49a5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:36,964 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,964 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,965 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b doesn't exist for region: bf24f627d705b5969ef5403995b49a5b on table testSequentialEditLogSeqNum 2024-11-24T04:48:36,966 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: bf24f627d705b5969ef5403995b49a5b on table testSequentialEditLogSeqNum 2024-11-24T04:48:36,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:48:36,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:48:36,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:48:36,990 INFO [StoreOpener-bf24f627d705b5969ef5403995b49a5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,992 INFO [StoreOpener-bf24f627d705b5969ef5403995b49a5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf24f627d705b5969ef5403995b49a5b columnFamilyName a 2024-11-24T04:48:36,992 DEBUG [StoreOpener-bf24f627d705b5969ef5403995b49a5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:36,993 INFO [StoreOpener-bf24f627d705b5969ef5403995b49a5b-1 {}] regionserver.HStore(327): Store=bf24f627d705b5969ef5403995b49a5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:36,993 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,994 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,994 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,995 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,995 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:36,998 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf24f627d705b5969ef5403995b49a5b 2024-11-24T04:48:37,005 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:37,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf24f627d705b5969ef5403995b49a5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61992382, jitterRate=-0.07624152302742004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:37,010 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf24f627d705b5969ef5403995b49a5b: Writing region info on filesystem at 1732423716964Initializing all the Stores at 1732423716990 (+26 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423716990Cleaning up temporary data from old regions at 1732423716995 (+5 ms)Region opened successfully at 1732423717010 (+15 ms) 2024-11-24T04:48:37,038 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bf24f627d705b5969ef5403995b49a5b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T04:48:37,074 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/.tmp/a/7a27b3ebb0984c36b708778a7032c217 is 81, key is testSequentialEditLogSeqNum/a:x0/1732423717010/Put/seqid=0 2024-11-24T04:48:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:48:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:48:37,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:48:37,100 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/.tmp/a/7a27b3ebb0984c36b708778a7032c217 2024-11-24T04:48:37,117 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/.tmp/a/7a27b3ebb0984c36b708778a7032c217 as hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/a/7a27b3ebb0984c36b708778a7032c217 2024-11-24T04:48:37,130 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/a/7a27b3ebb0984c36b708778a7032c217, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T04:48:37,132 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for bf24f627d705b5969ef5403995b49a5b in 94ms, sequenceid=13, compaction requested=false 2024-11-24T04:48:37,132 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bf24f627d705b5969ef5403995b49a5b: 2024-11-24T04:48:37,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:48:37,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:48:37,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:48:37,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:48:37,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:48:37,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741874_1051 (size=3256) 2024-11-24T04:48:37,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741874_1051 (size=3256) 2024-11-24T04:48:37,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741874_1051 (size=3256) 2024-11-24T04:48:37,172 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902, size=3.2 K (3256bytes) 2024-11-24T04:48:37,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902 2024-11-24T04:48:37,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902 after 0ms 2024-11-24T04:48:37,214 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:37,214 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902 took 42ms 2024-11-24T04:48:37,217 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902 so closing down 2024-11-24T04:48:37,217 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:37,218 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423716902.temp 2024-11-24T04:48:37,253 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000003-wal.1732423716902.temp 2024-11-24T04:48:37,257 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:37,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741877_1054 (size=2650) 2024-11-24T04:48:37,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741877_1054 (size=2650) 2024-11-24T04:48:37,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741877_1054 (size=2650) 2024-11-24T04:48:37,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000003-wal.1732423716902.temp (wrote 15 edits, skipped 0 edits in 4 ms) 2024-11-24T04:48:37,277 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000003-wal.1732423716902.temp to hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000020 2024-11-24T04:48:37,277 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 63 ms; skipped=2; WAL=hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-24T04:48:37,277 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902, journal: Splitting hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902, size=3.2 K (3256bytes) at 1732423717172Finishing writing output for hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902 so closing down at 1732423717217 (+45 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000003-wal.1732423716902.temp at 1732423717253 (+36 ms)3 split writer threads finished at 1732423717257 (+4 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000003-wal.1732423716902.temp (wrote 15 edits, skipped 0 edits in 4 ms) at 1732423717271 (+14 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000003-wal.1732423716902.temp to hdfs://localhost:43985/hbase/data/default/testSequentialEditLogSeqNum/bf24f627d705b5969ef5403995b49a5b/recovered.edits/0000000000000000020 at 1732423717277 (+6 ms)Processed 17 edits across 1 Regions in 63 ms; skipped=2; WAL=hdfs://localhost:43985/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423716894/wal.1732423716902, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732423717277 2024-11-24T04:48:37,294 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=404 (was 399) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:51760 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42514 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=588 (was 588), ProcessCount=11 (was 11), AvailableMemoryMB=10302 (was 10306) 2024-11-24T04:48:37,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=404, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=588, ProcessCount=11, AvailableMemoryMB=10301 2024-11-24T04:48:37,327 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:37,330 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:37,369 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:37,371 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53772009, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-53772009, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:37,386 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53772009/hregion-53772009.1732423717372, exclude list is [], retry=0 2024-11-24T04:48:37,390 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:37,390 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:37,394 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:37,423 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53772009/hregion-53772009.1732423717372 2024-11-24T04:48:37,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:48:37,424 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 268425678b16600497d712d935cb203d, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:37,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:48:37,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:48:37,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:48:37,440 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:37,442 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,449 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName a 2024-11-24T04:48:37,450 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:37,450 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:37,450 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,452 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName b 2024-11-24T04:48:37,452 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:37,452 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:37,452 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,455 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName c 2024-11-24T04:48:37,455 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:37,456 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:37,456 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,457 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d 2024-11-24T04:48:37,457 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d 2024-11-24T04:48:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,458 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,459 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:37,461 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,464 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:37,465 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 268425678b16600497d712d935cb203d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66191330, jitterRate=-0.013672322034835815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:37,466 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 268425678b16600497d712d935cb203d: Writing region info on filesystem at 1732423717440Initializing all the Stores at 1732423717441 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423717441Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423717442 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423717442Cleaning up temporary data from old regions at 1732423717458 (+16 ms)Region opened successfully at 1732423717466 (+8 ms) 2024-11-24T04:48:37,466 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 268425678b16600497d712d935cb203d, disabling compactions & flushes 2024-11-24T04:48:37,466 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:37,466 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:37,466 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. after waiting 0 ms 2024-11-24T04:48:37,466 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:37,468 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:37,468 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 268425678b16600497d712d935cb203d: Waiting for close lock at 1732423717466Disabling compacts and flushes for region at 1732423717466Disabling writes for close at 1732423717466Writing region close event to WAL at 1732423717468 (+2 ms)Closed at 1732423717468 2024-11-24T04:48:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741878_1055 (size=175) 2024-11-24T04:48:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741878_1055 (size=175) 2024-11-24T04:48:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741878_1055 (size=175) 2024-11-24T04:48:37,477 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:37,477 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53772009:(num 1732423717372) 2024-11-24T04:48:37,477 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:37,481 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:37,499 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482, exclude list is [], retry=0 2024-11-24T04:48:37,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:37,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:37,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:37,530 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 2024-11-24T04:48:37,530 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:37,530 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 268425678b16600497d712d935cb203d, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:37,530 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:37,530 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,531 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,533 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,535 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName a 2024-11-24T04:48:37,535 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:37,536 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:37,536 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,537 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName b 2024-11-24T04:48:37,537 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:37,538 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:37,538 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,539 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName c 2024-11-24T04:48:37,539 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:37,539 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:37,540 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,541 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d 2024-11-24T04:48:37,542 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d 2024-11-24T04:48:37,543 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,543 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,543 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:37,545 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 268425678b16600497d712d935cb203d 2024-11-24T04:48:37,546 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 268425678b16600497d712d935cb203d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62284685, jitterRate=-0.07188586890697479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:37,547 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 268425678b16600497d712d935cb203d: Writing region info on filesystem at 1732423717531Initializing all the Stores at 1732423717533 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423717533Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423717533Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423717533Cleaning up temporary data from old regions at 1732423717543 (+10 ms)Region opened successfully at 1732423717547 (+4 ms) 2024-11-24T04:48:37,553 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732423717551/Put/seqid=0 2024-11-24T04:48:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:48:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:48:37,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:48:37,564 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43985/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 268425678b16600497d712d935cb203d/a 2024-11-24T04:48:37,573 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T04:48:37,574 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:48:37,574 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 268425678b16600497d712d935cb203d: 2024-11-24T04:48:37,576 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_ 2024-11-24T04:48:37,577 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43985/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 268425678b16600497d712d935cb203d/a as hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_ - updating store file list. 2024-11-24T04:48:37,586 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T04:48:37,587 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_ into 268425678b16600497d712d935cb203d/a 2024-11-24T04:48:37,587 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43985/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 268425678b16600497d712d935cb203d/a (new location: hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_) 2024-11-24T04:48:37,640 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482, size=0 (0bytes) 2024-11-24T04:48:37,640 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 might be still open, length is 0 2024-11-24T04:48:37,640 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 2024-11-24T04:48:37,640 WARN [IPC Server handler 1 on default port 43985 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T04:48:37,641 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 after 1ms 2024-11-24T04:48:40,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35864 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43379:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35864 dst: /127.0.0.1:43379 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43379 remote=/127.0.0.1:35864]. Total timeout mills is 60000, 57098 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:40,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42740 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42740 dst: /127.0.0.1:35959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:40,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:51884 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45441:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51884 dst: /127.0.0.1:45441 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:40,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741880_1059 (size=606) 2024-11-24T04:48:40,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741880_1059 (size=606) 2024-11-24T04:48:41,641 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 after 4001ms 2024-11-24T04:48:41,676 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:41,676 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 took 4036ms 2024-11-24T04:48:41,689 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482; continuing. 2024-11-24T04:48:41,689 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 so closing down 2024-11-24T04:48:41,689 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:41,694 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732423717482.temp 2024-11-24T04:48:41,730 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005-wal.1732423717482.temp 2024-11-24T04:48:41,731 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:41,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741882_1060 (size=329) 2024-11-24T04:48:41,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741882_1060 (size=329) 2024-11-24T04:48:41,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741882_1060 (size=329) 2024-11-24T04:48:41,742 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005-wal.1732423717482.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:41,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005-wal.1732423717482.temp to hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005 2024-11-24T04:48:41,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 57 ms; skipped=1; WAL=hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:48:41,743 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482, journal: Splitting hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482, size=0 (0bytes) at 1732423717640Finishing writing output for hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 so closing down at 1732423721689 (+4049 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005-wal.1732423717482.temp at 1732423721730 (+41 ms)3 split writer threads finished at 1732423721731 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005-wal.1732423717482.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732423721742 (+11 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005-wal.1732423717482.temp to hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005 at 1732423721743 (+1 ms)Processed 2 edits across 1 Regions in 57 ms; skipped=1; WAL=hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482, size=0, length=0, corrupted=false, cancelled=false at 1732423721743 2024-11-24T04:48:41,745 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423717482 2024-11-24T04:48:41,746 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005 2024-11-24T04:48:41,746 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:41,749 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:41,767 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423721750, exclude list is [], retry=0 2024-11-24T04:48:41,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:41,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:41,771 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:41,809 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423721750 2024-11-24T04:48:41,812 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:41,812 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 268425678b16600497d712d935cb203d, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:41,813 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:41,813 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,813 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,815 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,816 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName a 2024-11-24T04:48:41,816 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:41,824 DEBUG [StoreFileOpener-268425678b16600497d712d935cb203d-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T04:48:41,824 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/9f2a7d8492494f6ba01ba260ddbf9bcf_SeqId_3_ 2024-11-24T04:48:41,825 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:41,825 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,826 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName b 2024-11-24T04:48:41,827 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:41,827 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:41,827 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,829 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 268425678b16600497d712d935cb203d columnFamilyName c 2024-11-24T04:48:41,829 DEBUG [StoreOpener-268425678b16600497d712d935cb203d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:41,829 INFO [StoreOpener-268425678b16600497d712d935cb203d-1 {}] regionserver.HStore(327): Store=268425678b16600497d712d935cb203d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:41,829 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,830 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d 2024-11-24T04:48:41,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d 2024-11-24T04:48:41,833 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005 2024-11-24T04:48:41,865 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:41,866 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005 2024-11-24T04:48:41,866 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 268425678b16600497d712d935cb203d 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T04:48:41,882 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/.tmp/a/3386087e7c3e480ba57bc54d8d1e24dd is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732423717594/Put/seqid=0 2024-11-24T04:48:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:48:41,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:48:41,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:48:41,901 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/.tmp/a/3386087e7c3e480ba57bc54d8d1e24dd 2024-11-24T04:48:41,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/.tmp/a/3386087e7c3e480ba57bc54d8d1e24dd as hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/3386087e7c3e480ba57bc54d8d1e24dd 2024-11-24T04:48:41,915 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/a/3386087e7c3e480ba57bc54d8d1e24dd, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T04:48:41,915 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 268425678b16600497d712d935cb203d in 49ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T04:48:41,916 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/0000000000000000005 2024-11-24T04:48:41,917 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,917 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,920 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:41,922 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 268425678b16600497d712d935cb203d 2024-11-24T04:48:41,926 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/268425678b16600497d712d935cb203d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T04:48:41,927 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 268425678b16600497d712d935cb203d; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67918838, jitterRate=0.012069553136825562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:41,927 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 268425678b16600497d712d935cb203d: Writing region info on filesystem at 1732423721813Initializing all the Stores at 1732423721814 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423721814Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423721815 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423721815Obtaining lock to block concurrent updates at 1732423721866 (+51 ms)Preparing flush snapshotting stores in 268425678b16600497d712d935cb203d at 1732423721866Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732423721866Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. at 1732423721866Flushing 268425678b16600497d712d935cb203d/a: creating writer at 1732423721866Flushing 268425678b16600497d712d935cb203d/a: appending metadata at 1732423721882 (+16 ms)Flushing 268425678b16600497d712d935cb203d/a: closing flushed file at 1732423721882Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bd10d2e: reopening flushed file at 1732423721907 (+25 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 268425678b16600497d712d935cb203d in 49ms, sequenceid=5, compaction requested=false; wal=null at 1732423721915 (+8 ms)Cleaning up temporary data from old regions at 1732423721917 (+2 ms)Region opened successfully at 1732423721927 (+10 ms) 2024-11-24T04:48:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 268425678b16600497d712d935cb203d, disabling compactions & flushes 2024-11-24T04:48:41,934 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. after waiting 0 ms 2024-11-24T04:48:41,934 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:41,936 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732423717328.268425678b16600497d712d935cb203d. 2024-11-24T04:48:41,936 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 268425678b16600497d712d935cb203d: Waiting for close lock at 1732423721933Disabling compacts and flushes for region at 1732423721933Disabling writes for close at 1732423721934 (+1 ms)Writing region close event to WAL at 1732423721936 (+2 ms)Closed at 1732423721936 2024-11-24T04:48:41,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741883_1061 (size=175) 2024-11-24T04:48:41,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741883_1061 (size=175) 2024-11-24T04:48:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741883_1061 (size=175) 2024-11-24T04:48:41,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:41,964 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423721750) 2024-11-24T04:48:41,978 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 404) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:43985 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1214887478_22 at /127.0.0.1:35880 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43985 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1214887478_22 at /127.0.0.1:42794 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=581 (was 588), ProcessCount=11 (was 11), AvailableMemoryMB=10297 (was 10301) 2024-11-24T04:48:41,990 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=581, ProcessCount=11, AvailableMemoryMB=10297 2024-11-24T04:48:42,005 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:42,010 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:48:42,014 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4464c5b832df,38057,1732423703360 2024-11-24T04:48:42,017 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6cdcf38d 2024-11-24T04:48:42,018 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:48:42,019 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:48:42,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:48:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T04:48:42,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:48:42,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T04:48:42,034 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:42,036 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:48:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:48:42,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:48:42,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:48:42,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:48:42,059 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269 2024-11-24T04:48:42,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:48:42,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:48:42,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:48:42,079 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:42,080 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:48:42,080 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,080 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,080 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:48:42,080 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,080 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,080 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423722079Disabling compacts and flushes for region at 1732423722079Disabling writes for close at 1732423722080 (+1 ms)Writing region close event to WAL at 1732423722080Closed at 1732423722080 2024-11-24T04:48:42,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:48:42,089 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732423722082"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732423722082"}]},"ts":"1732423722082"} 2024-11-24T04:48:42,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:48:42,094 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:48:42,097 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732423722094"}]},"ts":"1732423722094"} 2024-11-24T04:48:42,101 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T04:48:42,102 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4464c5b832df=0} racks are {/default-rack=0} 2024-11-24T04:48:42,103 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:48:42,103 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:48:42,103 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T04:48:42,103 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:48:42,103 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:48:42,104 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T04:48:42,104 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:48:42,104 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:48:42,104 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T04:48:42,104 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:48:42,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN}] 2024-11-24T04:48:42,108 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN 2024-11-24T04:48:42,109 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN; state=OFFLINE, location=4464c5b832df,44211,1732423704184; forceNewPlan=false, retain=false 2024-11-24T04:48:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:48:42,263 INFO [4464c5b832df:38057 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:48:42,263 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPENING, regionLocation=4464c5b832df,44211,1732423704184 2024-11-24T04:48:42,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN because future has completed 2024-11-24T04:48:42,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184}] 2024-11-24T04:48:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:48:42,421 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:48:42,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:48:42,429 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,430 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:42,430 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,430 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:42,430 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,430 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,432 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,433 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf1 2024-11-24T04:48:42,434 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:42,434 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:42,434 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,436 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf2 2024-11-24T04:48:42,436 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:42,437 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:42,437 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,438 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,438 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,439 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,439 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,440 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:48:42,443 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,446 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:42,448 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3002ca1159d88d4ded5fb6e5f70fe27e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73219615, jitterRate=0.09105728566646576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:48:42,448 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,449 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Running coprocessor pre-open hook at 1732423722431Writing region info on filesystem at 1732423722431Initializing all the Stores at 1732423722431Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423722432 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423722432Cleaning up temporary data from old regions at 1732423722439 (+7 ms)Running coprocessor post-open hooks at 1732423722448 (+9 ms)Region opened successfully at 1732423722449 (+1 ms) 2024-11-24T04:48:42,452 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., pid=6, masterSystemTime=1732423722421 2024-11-24T04:48:42,456 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,456 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,459 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPEN, openSeqNum=2, regionLocation=4464c5b832df,44211,1732423704184 2024-11-24T04:48:42,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184 because future has completed 2024-11-24T04:48:42,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T04:48:42,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184 in 201 msec 2024-11-24T04:48:42,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T04:48:42,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN in 367 msec 2024-11-24T04:48:42,482 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:48:42,482 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732423722482"}]},"ts":"1732423722482"} 2024-11-24T04:48:42,485 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T04:48:42,491 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:48:42,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 466 msec 2024-11-24T04:48:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:48:42,668 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T04:48:42,668 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T04:48:42,669 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:48:42,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T04:48:42,677 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:48:42,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T04:48:42,692 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=2] 2024-11-24T04:48:42,693 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:48:42,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:48:42,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=3002ca1159d88d4ded5fb6e5f70fe27e, source=4464c5b832df,44211,1732423704184, destination=4464c5b832df,42899,1732423704094, warming up region on 4464c5b832df,42899,1732423704094 2024-11-24T04:48:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:48:42,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=3002ca1159d88d4ded5fb6e5f70fe27e, source=4464c5b832df,44211,1732423704184, destination=4464c5b832df,42899,1732423704094, running balancer 2024-11-24T04:48:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE 2024-11-24T04:48:42,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41435, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:48:42,728 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE 2024-11-24T04:48:42,731 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=CLOSING, regionLocation=4464c5b832df,44211,1732423704184 2024-11-24T04:48:42,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE because future has completed 2024-11-24T04:48:42,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:42,738 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:48:42,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184}] 2024-11-24T04:48:42,740 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,747 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf1 2024-11-24T04:48:42,747 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:42,753 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:42,753 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,755 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf2 2024-11-24T04:48:42,755 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:42,756 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:48:42,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:48:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423722756Disabling compacts and flushes for region at 1732423722756Disabling writes for close at 1732423722756Writing region close event to WAL at 1732423722760 (+4 ms)Closed at 1732423722760 2024-11-24T04:48:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T04:48:42,897 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:42,897 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:48:42,898 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:48:42,898 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,898 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,898 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:48:42,898 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:42,898 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 3002ca1159d88d4ded5fb6e5f70fe27e 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T04:48:42,920 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/206df8d4293144a693b2e772569dce1f is 35, key is r1/cf1:q/1732423722697/Put/seqid=0 2024-11-24T04:48:42,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:48:42,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:48:42,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:48:43,339 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/206df8d4293144a693b2e772569dce1f 2024-11-24T04:48:43,349 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/206df8d4293144a693b2e772569dce1f as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f 2024-11-24T04:48:43,359 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T04:48:43,360 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3002ca1159d88d4ded5fb6e5f70fe27e in 462ms, sequenceid=5, compaction requested=false 2024-11-24T04:48:43,360 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T04:48:43,370 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T04:48:43,373 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:43,373 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423722898Running coprocessor pre-close hooks at 1732423722898Disabling compacts and flushes for region at 1732423722898Disabling writes for close at 1732423722898Obtaining lock to block concurrent updates at 1732423722898Preparing flush snapshotting stores in 3002ca1159d88d4ded5fb6e5f70fe27e at 1732423722898Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732423722899 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. at 1732423722899Flushing 3002ca1159d88d4ded5fb6e5f70fe27e/cf1: creating writer at 1732423722899Flushing 3002ca1159d88d4ded5fb6e5f70fe27e/cf1: appending metadata at 1732423722919 (+20 ms)Flushing 3002ca1159d88d4ded5fb6e5f70fe27e/cf1: closing flushed file at 1732423722919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5956e08: reopening flushed file at 1732423723346 (+427 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3002ca1159d88d4ded5fb6e5f70fe27e in 462ms, sequenceid=5, compaction requested=false at 1732423723360 (+14 ms)Writing region close event to WAL at 1732423723361 (+1 ms)Running coprocessor post-close hooks at 1732423723371 (+10 ms)Closed at 1732423723373 (+2 ms) 2024-11-24T04:48:43,374 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3002ca1159d88d4ded5fb6e5f70fe27e move to 4464c5b832df,42899,1732423704094 record at close sequenceid=5 2024-11-24T04:48:43,377 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=CLOSED 2024-11-24T04:48:43,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184 because future has completed 2024-11-24T04:48:43,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T04:48:43,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184 in 644 msec 2024-11-24T04:48:43,390 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE; state=CLOSED, location=4464c5b832df,42899,1732423704094; forceNewPlan=false, retain=false 2024-11-24T04:48:43,541 INFO [4464c5b832df:38057 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:48:43,541 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPENING, regionLocation=4464c5b832df,42899,1732423704094 2024-11-24T04:48:43,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE because future has completed 2024-11-24T04:48:43,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094}] 2024-11-24T04:48:43,702 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T04:48:43,702 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:43,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T04:48:43,703 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:43,717 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:43,717 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:43,718 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,718 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:43,720 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,720 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,724 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,726 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf1 2024-11-24T04:48:43,726 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:43,739 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f 2024-11-24T04:48:43,739 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:43,739 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,741 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf2 2024-11-24T04:48:43,741 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:43,741 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:43,742 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,744 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,746 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,747 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,747 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,748 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:48:43,749 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,750 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3002ca1159d88d4ded5fb6e5f70fe27e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67226589, jitterRate=0.0017542392015457153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:48:43,751 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:43,752 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Running coprocessor pre-open hook at 1732423723720Writing region info on filesystem at 1732423723720Initializing all the Stores at 1732423723724 (+4 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423723724Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423723724Cleaning up temporary data from old regions at 1732423723747 (+23 ms)Running coprocessor post-open hooks at 1732423723751 (+4 ms)Region opened successfully at 1732423723752 (+1 ms) 2024-11-24T04:48:43,753 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., pid=9, masterSystemTime=1732423723711 2024-11-24T04:48:43,756 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:43,756 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:43,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPEN, openSeqNum=9, regionLocation=4464c5b832df,42899,1732423704094 2024-11-24T04:48:43,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094 because future has completed 2024-11-24T04:48:43,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T04:48:43,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094 in 218 msec 2024-11-24T04:48:43,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE in 1.0440 sec 2024-11-24T04:48:43,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:48:43,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:48:43,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:46884 deadline: 1732423783800, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=42899 startCode=1732423704094. As of locationSeqNum=5. 2024-11-24T04:48:43,809 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=42899 startCode=1732423704094. As of locationSeqNum=5. 2024-11-24T04:48:43,810 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=42899 startCode=1732423704094. As of locationSeqNum=5. 2024-11-24T04:48:43,810 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,42899,1732423704094, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=42899 startCode=1732423704094. As of locationSeqNum=5. 2024-11-24T04:48:43,917 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:48:43,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:48:43,934 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3002ca1159d88d4ded5fb6e5f70fe27e 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T04:48:43,959 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/2766ec740fb74ca9af47b0353e7e84fb is 29, key is r1/cf1:/1732423723921/DeleteFamily/seqid=0 2024-11-24T04:48:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:48:43,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:48:43,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:48:43,995 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,020 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,040 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf2/1d82922b28c8440fbc52012520edd175 is 29, key is r1/cf2:/1732423723921/DeleteFamily/seqid=0 2024-11-24T04:48:44,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:48:44,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:48:44,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:48:44,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf2/1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,059 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/2766ec740fb74ca9af47b0353e7e84fb as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,069 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,070 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T04:48:44,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf2/1d82922b28c8440fbc52012520edd175 as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,081 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,081 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T04:48:44,082 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 3002ca1159d88d4ded5fb6e5f70fe27e in 148ms, sequenceid=12, compaction requested=false 2024-11-24T04:48:44,083 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3002ca1159d88d4ded5fb6e5f70fe27e: 2024-11-24T04:48:44,086 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T04:48:44,088 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3002ca1159d88d4ded5fb6e5f70fe27e/cf1 is initiating major compaction (all files) 2024-11-24T04:48:44,088 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:48:44,088 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:44,089 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3002ca1159d88d4ded5fb6e5f70fe27e/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,090 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f, hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb] into tmpdir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp, totalSize=9.5 K 2024-11-24T04:48:44,092 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 206df8d4293144a693b2e772569dce1f, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732423722697 2024-11-24T04:48:44,093 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2766ec740fb74ca9af47b0353e7e84fb, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T04:48:44,107 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3002ca1159d88d4ded5fb6e5f70fe27e#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T04:48:44,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:48:44,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:48:44,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:48:44,134 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf1/222b5e1105914c6ba6d87b6b1c32ae0c as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/222b5e1105914c6ba6d87b6b1c32ae0c 2024-11-24T04:48:44,154 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 3002ca1159d88d4ded5fb6e5f70fe27e/cf1 of 3002ca1159d88d4ded5fb6e5f70fe27e into 222b5e1105914c6ba6d87b6b1c32ae0c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:48:44,154 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3002ca1159d88d4ded5fb6e5f70fe27e: 2024-11-24T04:48:44,154 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T04:48:44,155 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3002ca1159d88d4ded5fb6e5f70fe27e/cf2 is initiating major compaction (all files) 2024-11-24T04:48:44,155 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:48:44,155 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:48:44,155 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3002ca1159d88d4ded5fb6e5f70fe27e/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,155 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175] into tmpdir=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp, totalSize=4.8 K 2024-11-24T04:48:44,157 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1d82922b28c8440fbc52012520edd175, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T04:48:44,181 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3002ca1159d88d4ded5fb6e5f70fe27e#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T04:48:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:48:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:48:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:48:44,237 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/.tmp/cf2/e80f7d88312e4767ae8e35d9b9512afa as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/e80f7d88312e4767ae8e35d9b9512afa 2024-11-24T04:48:44,247 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 3002ca1159d88d4ded5fb6e5f70fe27e/cf2 of 3002ca1159d88d4ded5fb6e5f70fe27e into e80f7d88312e4767ae8e35d9b9512afa(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:48:44,247 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3002ca1159d88d4ded5fb6e5f70fe27e: 2024-11-24T04:48:44,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=3002ca1159d88d4ded5fb6e5f70fe27e, source=4464c5b832df,42899,1732423704094, destination=4464c5b832df,44211,1732423704184, warming up region on 4464c5b832df,44211,1732423704184 2024-11-24T04:48:44,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=3002ca1159d88d4ded5fb6e5f70fe27e, source=4464c5b832df,42899,1732423704094, destination=4464c5b832df,44211,1732423704184, running balancer 2024-11-24T04:48:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE 2024-11-24T04:48:44,255 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE 2024-11-24T04:48:44,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:44,258 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,259 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=CLOSING, regionLocation=4464c5b832df,42899,1732423704094 2024-11-24T04:48:44,260 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf1 2024-11-24T04:48:44,260 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:44,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE because future has completed 2024-11-24T04:48:44,262 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:48:44,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094}] 2024-11-24T04:48:44,272 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f 2024-11-24T04:48:44,279 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/222b5e1105914c6ba6d87b6b1c32ae0c 2024-11-24T04:48:44,286 INFO [StoreFileOpener-3002ca1159d88d4ded5fb6e5f70fe27e-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,286 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,286 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:44,286 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,288 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf2 2024-11-24T04:48:44,288 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:44,301 INFO [StoreFileOpener-3002ca1159d88d4ded5fb6e5f70fe27e-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,301 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T04:48:44,313 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/e80f7d88312e4767ae8e35d9b9512afa 2024-11-24T04:48:44,313 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:48:44,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:48:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44211 {}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423724314Disabling compacts and flushes for region at 1732423724314Disabling writes for close at 1732423724314Writing region close event to WAL at 1732423724317 (+3 ms)Closed at 1732423724317 2024-11-24T04:48:44,418 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,418 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:48:44,418 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:48:44,418 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,418 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,419 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:48:44,419 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,419 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f, hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb] to archive 2024-11-24T04:48:44,426 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:48:44,437 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f to hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/206df8d4293144a693b2e772569dce1f 2024-11-24T04:48:44,438 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb to hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/2766ec740fb74ca9af47b0353e7e84fb 2024-11-24T04:48:44,456 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175] to archive 2024-11-24T04:48:44,458 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:48:44,461 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175 to hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/1d82922b28c8440fbc52012520edd175 2024-11-24T04:48:44,473 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T04:48:44,474 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,474 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423724418Running coprocessor pre-close hooks at 1732423724418Disabling compacts and flushes for region at 1732423724418Disabling writes for close at 1732423724419 (+1 ms)Writing region close event to WAL at 1732423724464 (+45 ms)Running coprocessor post-close hooks at 1732423724473 (+9 ms)Closed at 1732423724473 2024-11-24T04:48:44,474 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 3002ca1159d88d4ded5fb6e5f70fe27e move to 4464c5b832df,44211,1732423704184 record at close sequenceid=12 2024-11-24T04:48:44,479 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=CLOSED 2024-11-24T04:48:44,480 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094 because future has completed 2024-11-24T04:48:44,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T04:48:44,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094 in 226 msec 2024-11-24T04:48:44,494 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE; state=CLOSED, location=4464c5b832df,44211,1732423704184; forceNewPlan=false, retain=false 2024-11-24T04:48:44,645 INFO [4464c5b832df:38057 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:48:44,645 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPENING, regionLocation=4464c5b832df,44211,1732423704184 2024-11-24T04:48:44,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE because future has completed 2024-11-24T04:48:44,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184}] 2024-11-24T04:48:44,805 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,805 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:44,806 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,806 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:44,806 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,806 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,807 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,808 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf1 2024-11-24T04:48:44,808 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:44,817 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/222b5e1105914c6ba6d87b6b1c32ae0c 2024-11-24T04:48:44,817 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:44,817 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,818 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf2 2024-11-24T04:48:44,818 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:44,829 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/e80f7d88312e4767ae8e35d9b9512afa 2024-11-24T04:48:44,829 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:44,829 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,830 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,833 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,834 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,834 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,835 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:48:44,837 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,838 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3002ca1159d88d4ded5fb6e5f70fe27e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62588993, jitterRate=-0.0673513263463974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:48:44,838 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,839 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Running coprocessor pre-open hook at 1732423724806Writing region info on filesystem at 1732423724806Initializing all the Stores at 1732423724807 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423724807Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423724807Cleaning up temporary data from old regions at 1732423724834 (+27 ms)Running coprocessor post-open hooks at 1732423724838 (+4 ms)Region opened successfully at 1732423724839 (+1 ms) 2024-11-24T04:48:44,840 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., pid=12, masterSystemTime=1732423724801 2024-11-24T04:48:44,843 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,843 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,844 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPEN, openSeqNum=18, regionLocation=4464c5b832df,44211,1732423704184 2024-11-24T04:48:44,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184 because future has completed 2024-11-24T04:48:44,848 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38057 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=4464c5b832df,44211,1732423704184, table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T04:48:44,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T04:48:44,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,44211,1732423704184 in 200 msec 2024-11-24T04:48:44,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, REOPEN/MOVE in 599 msec 2024-11-24T04:48:44,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:48:44,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:48:44,862 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 4464c5b832df,44211,1732423704184: testing ***** 2024-11-24T04:48:44,862 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T04:48:44,864 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T04:48:44,866 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T04:48:44,874 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T04:48:44,876 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T04:48:44,894 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 481887288 }, "NonHeapMemoryUsage": { "committed": 172228608, "init": 7667712, "max": -1, "used": 169621848 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "4464c5b832df", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11067, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 65, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 33, "ProcessCallTime_98th_percentile": 65, "ProcessCallTime_99th_percentile": 65, "ProcessCallTime_99.9th_percentile": 65, "ProcessCallTime_TimeRangeCount_0-1": 12, "ProcessCallTime_TimeRangeCount_1-3": 11, "ProcessCallTime_TimeRangeCount_3-10": 6, "ProcessCallTime_TimeRangeCount_30-100": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 65, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 33, "TotalCallTime_98th_percentile": 65, "TotalCallTime_99th_percentile": 65, "TotalCallTime_99.9th_percentile": 65, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 6, "TotalCallTime_TimeRangeCount_30-100": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 158, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5745 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "4464c5b832df", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57729", "tag.serverName": "4464c5b832df,44211,1732423704184", "tag.clusterId": "894908be-bd12-4c70-96f1-bb976b7347b7", "tag.Context": "regionserver", "tag.Hostname": "4464c5b832df", "regionCount": 1, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732423704184, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 4, "Put_max": 4, "Put_mean": 4, "Put_25th_percentile": 4, "Put_median": 4, "Put_75th_percentile": 4, "Put_90th_percentile": 4, "Put_95th_percentile": 4, "Put_98th_percentile": 4, "Put_99th_percentile": 4, "Put_99.9th_percentile": 4, "Put_TimeRangeCount_3-10": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 462, "FlushTime_max": 462, "FlushTime_mean": 462, "FlushTime_25th_percentile": 462, "FlushTime_median": 462, "FlushTime_75th_percentile": 462, "FlushTime_90th_percentile": 462, "FlushTime_95th_percentile": 462, "FlushTime_98th_percentile": 462, "FlushTime_99th_percentile": 462, "FlushTime_99.9th_percentile": 462, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T04:48:44,900 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38057 {}] master.MasterRpcServices(700): 4464c5b832df,44211,1732423704184 reported a fatal error: ***** ABORTING region server 4464c5b832df,44211,1732423704184: testing ***** 2024-11-24T04:48:44,902 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4464c5b832df,44211,1732423704184' ***** 2024-11-24T04:48:44,902 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T04:48:44,902 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:48:44,903 INFO [RS:1;4464c5b832df:44211 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T04:48:44,903 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:48:44,903 INFO [RS:1;4464c5b832df:44211 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T04:48:44,903 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(3091): Received CLOSE for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,903 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(956): aborting server 4464c5b832df,44211,1732423704184 2024-11-24T04:48:44,903 INFO [RS:1;4464c5b832df:44211 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:48:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:45590 deadline: 1732423784903, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=44211 startCode=1732423704184. As of locationSeqNum=12. 2024-11-24T04:48:44,904 INFO [RS:1;4464c5b832df:44211 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4464c5b832df:44211. 2024-11-24T04:48:44,904 DEBUG [RS:1;4464c5b832df:44211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:48:44,904 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:48:44,904 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,904 DEBUG [RS:1;4464c5b832df:44211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:48:44,904 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,904 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:48:44,904 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,904 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:48:44,905 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1325): Online Regions={3002ca1159d88d4ded5fb6e5f70fe27e=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.} 2024-11-24T04:48:44,905 DEBUG [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1351): Waiting on 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:44,906 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,42899,1732423704094, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,42899,1732423704094, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=44211 startCode=1732423704184. As of locationSeqNum=12. 2024-11-24T04:48:44,906 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,42899,1732423704094, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=44211 startCode=1732423704184. As of locationSeqNum=12. 2024-11-24T04:48:44,906 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,42899,1732423704094, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=44211 startCode=1732423704184. As of locationSeqNum=12. 2024-11-24T04:48:44,909 INFO [regionserver/4464c5b832df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:48:44,920 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:44,920 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423724904Running coprocessor pre-close hooks at 1732423724904Disabling compacts and flushes for region at 1732423724904Disabling writes for close at 1732423724904Writing region close event to WAL at 1732423724919 (+15 ms)Running coprocessor post-close hooks at 1732423724919Closed at 1732423724920 (+1 ms) 2024-11-24T04:48:44,920 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:45,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4464c5b832df,44211,1732423704184 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:45,018 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4464c5b832df,44211,1732423704184 aborting 2024-11-24T04:48:45,018 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4464c5b832df,44211,1732423704184 aborting 2024-11-24T04:48:45,019 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=12 from cache 2024-11-24T04:48:45,105 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(976): stopping server 4464c5b832df,44211,1732423704184; all regions closed. 2024-11-24T04:48:45,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741834_1010 (size=2288) 2024-11-24T04:48:45,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741834_1010 (size=2288) 2024-11-24T04:48:45,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741834_1010 (size=2288) 2024-11-24T04:48:45,112 DEBUG [RS:1;4464c5b832df:44211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:48:45,112 INFO [RS:1;4464c5b832df:44211 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:48:45,112 INFO [RS:1;4464c5b832df:44211 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:48:45,113 INFO [RS:1;4464c5b832df:44211 {}] hbase.ChoreService(370): Chore service for: regionserver/4464c5b832df:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T04:48:45,113 INFO [regionserver/4464c5b832df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:48:45,113 INFO [RS:1;4464c5b832df:44211 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:48:45,113 INFO [RS:1;4464c5b832df:44211 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:48:45,113 INFO [RS:1;4464c5b832df:44211 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:48:45,113 INFO [RS:1;4464c5b832df:44211 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:48:45,113 INFO [RS:1;4464c5b832df:44211 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44211 2024-11-24T04:48:45,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:48:45,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4464c5b832df,44211,1732423704184 2024-11-24T04:48:45,179 INFO [RS:1;4464c5b832df:44211 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:48:45,179 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4464c5b832df,44211,1732423704184] 2024-11-24T04:48:45,199 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4464c5b832df,44211,1732423704184 already deleted, retry=false 2024-11-24T04:48:45,199 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4464c5b832df,44211,1732423704184 on 4464c5b832df,38057,1732423703360 2024-11-24T04:48:45,205 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4464c5b832df,44211,1732423704184, splitWal=true, meta=false 2024-11-24T04:48:45,207 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4464c5b832df,44211,1732423704184, splitWal=true, meta=false 2024-11-24T04:48:45,207 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 4464c5b832df,44211,1732423704184 (carryingMeta=false) 4464c5b832df,44211,1732423704184/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1cbd8d48[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T04:48:45,210 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): 4464c5b832df,44211,1732423704184 had 1 regions 2024-11-24T04:48:45,212 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4464c5b832df,44211,1732423704184, splitWal=true, meta=false, isMeta: false 2024-11-24T04:48:45,215 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting 2024-11-24T04:48:45,216 INFO [PEWorker-5 {}] master.SplitWALManager(105): 4464c5b832df,44211,1732423704184 WAL count=1, meta=false 2024-11-24T04:48:45,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 4464c5b832df%2C44211%2C1732423704184.1732423706054}] 2024-11-24T04:48:45,225 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=4464c5b832df,42899,1732423704094 2024-11-24T04:48:45,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 4464c5b832df%2C44211%2C1732423704184.1732423706054, worker=4464c5b832df,42899,1732423704094}] 2024-11-24T04:48:45,239 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18] 2024-11-24T04:48:45,241 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4464c5b832df:44211 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 4464c5b832df/172.17.0.2:44211 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:45,241 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18, error=java.net.ConnectException: Call to address=4464c5b832df:44211 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 4464c5b832df/172.17.0.2:44211 2024-11-24T04:48:45,242 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-24T04:48:45,242 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18 from cache 2024-11-24T04:48:45,242 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 4464c5b832df:44211 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 4464c5b832df/172.17.0.2:44211 2024-11-24T04:48:45,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:48:45,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44211-0x1016b2bd3dc0002, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:48:45,290 INFO [RS:1;4464c5b832df:44211 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:48:45,290 INFO [RS:1;4464c5b832df:44211 {}] regionserver.HRegionServer(1031): Exiting; stopping=4464c5b832df,44211,1732423704184; zookeeper connection closed. 2024-11-24T04:48:45,290 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39d10538 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39d10538 2024-11-24T04:48:45,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T04:48:45,418 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054, size=2.2 K (2288bytes) 2024-11-24T04:48:45,418 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 2024-11-24T04:48:45,419 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 after 1ms 2024-11-24T04:48:45,454 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:45,454 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 took 36ms 2024-11-24T04:48:45,464 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 3002ca1159d88d4ded5fb6e5f70fe27e: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T04:48:45,464 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 so closing down 2024-11-24T04:48:45,464 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:45,464 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:45,464 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-24T04:48:45,464 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054, journal: Splitting hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054, size=2.2 K (2288bytes) at 1732423725418Finishing writing output for hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 so closing down at 1732423725464 (+46 ms)3 split writer threads finished at 1732423725464Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732423725464 2024-11-24T04:48:45,464 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 2024-11-24T04:48:45,466 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T04:48:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38057 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T04:48:45,475 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting/4464c5b832df%2C44211%2C1732423704184.1732423706054 to hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs 2024-11-24T04:48:45,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T04:48:45,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4464c5b832df%2C44211%2C1732423704184.1732423706054, worker=4464c5b832df,42899,1732423704094 in 247 msec 2024-11-24T04:48:45,483 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=4464c5b832df,42899,1732423704094 2024-11-24T04:48:45,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T04:48:45,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 4464c5b832df%2C44211%2C1732423704184.1732423706054, worker=4464c5b832df,42899,1732423704094 in 266 msec 2024-11-24T04:48:45,489 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting dir is empty, no logs to split. 2024-11-24T04:48:45,489 INFO [PEWorker-5 {}] master.SplitWALManager(105): 4464c5b832df,44211,1732423704184 WAL count=0, meta=false 2024-11-24T04:48:45,489 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 4464c5b832df,44211,1732423704184 WAL splitting is done? wals=0, meta=false 2024-11-24T04:48:45,494 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 4464c5b832df,44211,1732423704184 failed, ignore...File hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/WALs/4464c5b832df,44211,1732423704184-splitting does not exist. 2024-11-24T04:48:45,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN}] 2024-11-24T04:48:45,505 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN 2024-11-24T04:48:45,506 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T04:48:45,550 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18] 2024-11-24T04:48:45,550 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 4464c5b832df:44211 this server is in the failed servers list 2024-11-24T04:48:45,551 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=4464c5b832df:44211 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 4464c5b832df:44211 2024-11-24T04:48:45,551 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 4464c5b832df:44211 2024-11-24T04:48:45,551 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,44211,1732423704184, seqNum=18 from cache 2024-11-24T04:48:45,657 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(204): Hosts are {4464c5b832df=0} racks are {/default-rack=0} 2024-11-24T04:48:45,657 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:48:45,657 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:48:45,657 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:48:45,658 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:48:45,658 INFO [4464c5b832df:38057 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:48:45,658 INFO [4464c5b832df:38057 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:48:45,658 DEBUG [4464c5b832df:38057 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:48:45,658 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPENING, regionLocation=4464c5b832df,42899,1732423704094 2024-11-24T04:48:45,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN because future has completed 2024-11-24T04:48:45,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094}] 2024-11-24T04:48:45,817 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:45,817 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 3002ca1159d88d4ded5fb6e5f70fe27e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:45,818 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,818 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:45,818 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,818 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,820 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,821 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf1 2024-11-24T04:48:45,821 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:45,828 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf1/222b5e1105914c6ba6d87b6b1c32ae0c 2024-11-24T04:48:45,828 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:45,828 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,829 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3002ca1159d88d4ded5fb6e5f70fe27e columnFamilyName cf2 2024-11-24T04:48:45,829 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:45,837 DEBUG [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/cf2/e80f7d88312e4767ae8e35d9b9512afa 2024-11-24T04:48:45,837 INFO [StoreOpener-3002ca1159d88d4ded5fb6e5f70fe27e-1 {}] regionserver.HStore(327): Store=3002ca1159d88d4ded5fb6e5f70fe27e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:45,837 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,838 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,839 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,840 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,840 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,840 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:48:45,842 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,842 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 3002ca1159d88d4ded5fb6e5f70fe27e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66204489, jitterRate=-0.013476237654685974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:48:45,843 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:48:45,843 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Running coprocessor pre-open hook at 1732423725818Writing region info on filesystem at 1732423725818Initializing all the Stores at 1732423725820 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423725820Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423725820Cleaning up temporary data from old regions at 1732423725840 (+20 ms)Running coprocessor post-open hooks at 1732423725843 (+3 ms)Region opened successfully at 1732423725843 2024-11-24T04:48:45,849 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., pid=17, masterSystemTime=1732423725813 2024-11-24T04:48:45,852 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:45,852 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:48:45,853 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3002ca1159d88d4ded5fb6e5f70fe27e, regionState=OPEN, openSeqNum=18, regionLocation=4464c5b832df,42899,1732423704094 2024-11-24T04:48:45,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094 because future has completed 2024-11-24T04:48:45,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T04:48:45,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3002ca1159d88d4ded5fb6e5f70fe27e, server=4464c5b832df,42899,1732423704094 in 198 msec 2024-11-24T04:48:45,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T04:48:45,867 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 4464c5b832df,44211,1732423704184 after splitting done 2024-11-24T04:48:45,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3002ca1159d88d4ded5fb6e5f70fe27e, ASSIGN in 366 msec 2024-11-24T04:48:45,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 4464c5b832df,44211,1732423704184, splitWal=true, meta=false in 667 msec 2024-11-24T04:48:46,059 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e., hostname=4464c5b832df,42899,1732423704094, seqNum=18] 2024-11-24T04:48:46,077 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 412), OpenFileDescriptor=1046 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=581 (was 581), ProcessCount=11 (was 11), AvailableMemoryMB=10263 (was 10297) 2024-11-24T04:48:46,079 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-11-24T04:48:46,094 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1046, MaxFileDescriptor=1048576, SystemLoadAverage=581, ProcessCount=11, AvailableMemoryMB=10263 2024-11-24T04:48:46,094 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-11-24T04:48:46,111 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:46,113 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:46,114 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:46,116 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91977073, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-91977073, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:46,134 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91977073/hregion-91977073.1732423726117, exclude list is [], retry=0 2024-11-24T04:48:46,137 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:46,137 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:46,138 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:46,176 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91977073/hregion-91977073.1732423726117 2024-11-24T04:48:46,176 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:48:46,177 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2e621772f03ae186fd273064d0484755, NAME => 'testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:48:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:48:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:48:46,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:46,202 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,203 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName a 2024-11-24T04:48:46,203 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,204 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,204 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,206 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName b 2024-11-24T04:48:46,206 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,207 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,207 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,209 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName c 2024-11-24T04:48:46,209 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,210 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,210 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,213 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,213 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,213 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:46,215 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,218 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:46,218 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2e621772f03ae186fd273064d0484755; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71671984, jitterRate=0.06799578666687012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:46,219 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2e621772f03ae186fd273064d0484755: Writing region info on filesystem at 1732423726200Initializing all the Stores at 1732423726201 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726201Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726201Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726201Cleaning up temporary data from old regions at 1732423726213 (+12 ms)Region opened successfully at 1732423726219 (+6 ms) 2024-11-24T04:48:46,219 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2e621772f03ae186fd273064d0484755, disabling compactions & flushes 2024-11-24T04:48:46,220 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,220 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,220 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. after waiting 0 ms 2024-11-24T04:48:46,220 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,221 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,221 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2e621772f03ae186fd273064d0484755: Waiting for close lock at 1732423726219Disabling compacts and flushes for region at 1732423726219Disabling writes for close at 1732423726220 (+1 ms)Writing region close event to WAL at 1732423726221 (+1 ms)Closed at 1732423726221 2024-11-24T04:48:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741892_1070 (size=175) 2024-11-24T04:48:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741892_1070 (size=175) 2024-11-24T04:48:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741892_1070 (size=175) 2024-11-24T04:48:46,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:46,229 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91977073:(num 1732423726117) 2024-11-24T04:48:46,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:46,231 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:46,249 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232, exclude list is [], retry=0 2024-11-24T04:48:46,252 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:46,252 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:46,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:46,294 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 2024-11-24T04:48:46,294 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:48:46,294 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e621772f03ae186fd273064d0484755, NAME => 'testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:46,295 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:46,295 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,295 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,299 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,300 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName a 2024-11-24T04:48:46,301 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,301 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,301 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,302 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName b 2024-11-24T04:48:46,302 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,302 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,302 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,303 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName c 2024-11-24T04:48:46,303 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,303 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,308 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,309 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,309 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,310 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:46,313 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,314 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2e621772f03ae186fd273064d0484755; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66416523, jitterRate=-0.01031668484210968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:46,315 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2e621772f03ae186fd273064d0484755: Writing region info on filesystem at 1732423726295Initializing all the Stores at 1732423726297 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726297Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726299 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726299Cleaning up temporary data from old regions at 1732423726309 (+10 ms)Region opened successfully at 1732423726315 (+6 ms) 2024-11-24T04:48:46,392 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2e621772f03ae186fd273064d0484755 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T04:48:46,418 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/a/8d7d318a5de84431bd51a0da168c0a67 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732423726315/Put/seqid=0 2024-11-24T04:48:46,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:48:46,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:48:46,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:48:46,457 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/a/8d7d318a5de84431bd51a0da168c0a67 2024-11-24T04:48:46,512 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/b/876d607f886e49e7bfdad326cb7835c7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732423726340/Put/seqid=0 2024-11-24T04:48:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:48:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:48:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:48:46,541 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/b/876d607f886e49e7bfdad326cb7835c7 2024-11-24T04:48:46,565 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/c/ea6737ed23b24dcab990dec51917260a is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732423726361/Put/seqid=0 2024-11-24T04:48:46,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:48:46,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:48:46,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:48:46,580 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/c/ea6737ed23b24dcab990dec51917260a 2024-11-24T04:48:46,588 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/a/8d7d318a5de84431bd51a0da168c0a67 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/a/8d7d318a5de84431bd51a0da168c0a67 2024-11-24T04:48:46,598 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/a/8d7d318a5de84431bd51a0da168c0a67, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:48:46,601 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/b/876d607f886e49e7bfdad326cb7835c7 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/b/876d607f886e49e7bfdad326cb7835c7 2024-11-24T04:48:46,607 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/b/876d607f886e49e7bfdad326cb7835c7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:48:46,608 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/c/ea6737ed23b24dcab990dec51917260a as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/c/ea6737ed23b24dcab990dec51917260a 2024-11-24T04:48:46,614 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/c/ea6737ed23b24dcab990dec51917260a, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:48:46,616 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2e621772f03ae186fd273064d0484755 in 225ms, sequenceid=33, compaction requested=false 2024-11-24T04:48:46,616 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2e621772f03ae186fd273064d0484755: 2024-11-24T04:48:46,617 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2e621772f03ae186fd273064d0484755, disabling compactions & flushes 2024-11-24T04:48:46,617 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,617 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,617 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. after waiting 0 ms 2024-11-24T04:48:46,617 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,629 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. 2024-11-24T04:48:46,629 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2e621772f03ae186fd273064d0484755: Waiting for close lock at 1732423726617Disabling compacts and flushes for region at 1732423726617Disabling writes for close at 1732423726617Writing region close event to WAL at 1732423726629 (+12 ms)Closed at 1732423726629 2024-11-24T04:48:46,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741894_1072 (size=6311) 2024-11-24T04:48:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741894_1072 (size=6311) 2024-11-24T04:48:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741894_1072 (size=6311) 2024-11-24T04:48:46,639 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/b/876d607f886e49e7bfdad326cb7835c7 to hdfs://localhost:43985/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/b/876d607f886e49e7bfdad326cb7835c7 2024-11-24T04:48:46,660 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232, size=6.2 K (6311bytes) 2024-11-24T04:48:46,660 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 2024-11-24T04:48:46,661 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 after 1ms 2024-11-24T04:48:46,697 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:46,697 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 took 37ms 2024-11-24T04:48:46,700 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 so closing down 2024-11-24T04:48:46,700 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:46,703 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423726232.temp 2024-11-24T04:48:46,739 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000003-wal.1732423726232.temp 2024-11-24T04:48:46,740 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:46,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741898_1076 (size=5575) 2024-11-24T04:48:46,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741898_1076 (size=5575) 2024-11-24T04:48:46,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741898_1076 (size=5575) 2024-11-24T04:48:46,754 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000003-wal.1732423726232.temp (wrote 30 edits, skipped 0 edits in 1 ms) 2024-11-24T04:48:46,755 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000003-wal.1732423726232.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000032 2024-11-24T04:48:46,755 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 57 ms; skipped=2; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-24T04:48:46,756 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232, journal: Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232, size=6.2 K (6311bytes) at 1732423726660Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 so closing down at 1732423726700 (+40 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000003-wal.1732423726232.temp at 1732423726739 (+39 ms)3 split writer threads finished at 1732423726740 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000003-wal.1732423726232.temp (wrote 30 edits, skipped 0 edits in 1 ms) at 1732423726754 (+14 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000003-wal.1732423726232.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000032 at 1732423726755 (+1 ms)Processed 32 edits across 1 Regions in 57 ms; skipped=2; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732423726756 (+1 ms) 2024-11-24T04:48:46,757 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726232 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423726232 2024-11-24T04:48:46,758 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000032 2024-11-24T04:48:46,758 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:46,760 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:46,778 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726761, exclude list is [], retry=0 2024-11-24T04:48:46,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:46,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:46,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:46,818 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726761 2024-11-24T04:48:46,819 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:46,819 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e621772f03ae186fd273064d0484755, NAME => 'testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:46,819 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:46,819 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,819 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,821 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,821 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName a 2024-11-24T04:48:46,822 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,827 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/a/8d7d318a5de84431bd51a0da168c0a67 2024-11-24T04:48:46,827 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,827 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,829 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName b 2024-11-24T04:48:46,829 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,829 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,829 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,830 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e621772f03ae186fd273064d0484755 columnFamilyName c 2024-11-24T04:48:46,830 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:46,837 DEBUG [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/c/ea6737ed23b24dcab990dec51917260a 2024-11-24T04:48:46,837 INFO [StoreOpener-2e621772f03ae186fd273064d0484755-1 {}] regionserver.HStore(327): Store=2e621772f03ae186fd273064d0484755/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:46,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,840 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000032 2024-11-24T04:48:46,877 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:46,879 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000032 2024-11-24T04:48:46,879 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2e621772f03ae186fd273064d0484755 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T04:48:46,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/b/29e4afd9c5b140b8b63f384cf738c388 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732423726340/Put/seqid=0 2024-11-24T04:48:46,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:48:46,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:48:46,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:48:46,908 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/b/29e4afd9c5b140b8b63f384cf738c388 2024-11-24T04:48:46,915 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/.tmp/b/29e4afd9c5b140b8b63f384cf738c388 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/b/29e4afd9c5b140b8b63f384cf738c388 2024-11-24T04:48:46,921 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/b/29e4afd9c5b140b8b63f384cf738c388, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T04:48:46,921 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2e621772f03ae186fd273064d0484755 in 42ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T04:48:46,922 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/0000000000000000032 2024-11-24T04:48:46,923 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,923 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,923 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:46,925 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2e621772f03ae186fd273064d0484755 2024-11-24T04:48:46,926 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/2e621772f03ae186fd273064d0484755/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T04:48:46,927 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2e621772f03ae186fd273064d0484755; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59109464, jitterRate=-0.11920034885406494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:46,928 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2e621772f03ae186fd273064d0484755: Writing region info on filesystem at 1732423726819Initializing all the Stores at 1732423726820 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726820Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726821 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423726821Obtaining lock to block concurrent updates at 1732423726879 (+58 ms)Preparing flush snapshotting stores in 2e621772f03ae186fd273064d0484755 at 1732423726879Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732423726879Flushing stores of testReplayEditsWrittenViaHRegion,,1732423726112.2e621772f03ae186fd273064d0484755. at 1732423726879Flushing 2e621772f03ae186fd273064d0484755/b: creating writer at 1732423726879Flushing 2e621772f03ae186fd273064d0484755/b: appending metadata at 1732423726895 (+16 ms)Flushing 2e621772f03ae186fd273064d0484755/b: closing flushed file at 1732423726895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16b85df7: reopening flushed file at 1732423726913 (+18 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2e621772f03ae186fd273064d0484755 in 42ms, sequenceid=32, compaction requested=false; wal=null at 1732423726921 (+8 ms)Cleaning up temporary data from old regions at 1732423726923 (+2 ms)Region opened successfully at 1732423726928 (+5 ms) 2024-11-24T04:48:46,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 406) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35880 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:38000 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:36546 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:33620 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:33722 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:42794 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1122 (was 1046) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=607 (was 581) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10258 (was 10263) 2024-11-24T04:48:46,961 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-11-24T04:48:46,977 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1122, MaxFileDescriptor=1048576, SystemLoadAverage=607, ProcessCount=11, AvailableMemoryMB=10257 2024-11-24T04:48:46,977 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-11-24T04:48:46,993 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:46,996 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:46,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:47,001 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79359956, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-79359956, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:47,016 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79359956/hregion-79359956.1732423727001, exclude list is [], retry=0 2024-11-24T04:48:47,020 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:47,025 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:47,029 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:47,079 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79359956/hregion-79359956.1732423727001 2024-11-24T04:48:47,091 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:47,092 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2a212a86e759e7e603b57c25e3c47677, NAME => 'testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:47,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:48:47,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:48:47,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:48:47,135 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:47,138 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,141 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName a 2024-11-24T04:48:47,141 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:47,142 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:47,142 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,154 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName b 2024-11-24T04:48:47,154 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:47,155 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:47,155 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,163 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName c 2024-11-24T04:48:47,164 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:47,164 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:47,165 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,166 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,166 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,170 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,170 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,171 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:47,172 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,179 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:47,180 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a212a86e759e7e603b57c25e3c47677; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62426306, jitterRate=-0.0697755515575409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:47,181 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a212a86e759e7e603b57c25e3c47677: Writing region info on filesystem at 1732423727135Initializing all the Stores at 1732423727137 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423727137Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423727137Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423727137Cleaning up temporary data from old regions at 1732423727170 (+33 ms)Region opened successfully at 1732423727181 (+11 ms) 2024-11-24T04:48:47,181 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2a212a86e759e7e603b57c25e3c47677, disabling compactions & flushes 2024-11-24T04:48:47,181 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:47,181 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:47,181 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. after waiting 0 ms 2024-11-24T04:48:47,181 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:47,184 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:47,184 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2a212a86e759e7e603b57c25e3c47677: Waiting for close lock at 1732423727181Disabling compacts and flushes for region at 1732423727181Disabling writes for close at 1732423727181Writing region close event to WAL at 1732423727184 (+3 ms)Closed at 1732423727184 2024-11-24T04:48:47,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741901_1079 (size=175) 2024-11-24T04:48:47,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741901_1079 (size=175) 2024-11-24T04:48:47,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741901_1079 (size=175) 2024-11-24T04:48:47,194 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:47,194 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79359956:(num 1732423727001) 2024-11-24T04:48:47,194 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:47,201 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:47,220 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202, exclude list is [], retry=0 2024-11-24T04:48:47,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:47,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:47,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:47,273 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 2024-11-24T04:48:47,273 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:47,379 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a212a86e759e7e603b57c25e3c47677, NAME => 'testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:47,382 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,382 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:47,383 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,383 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,386 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,396 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName a 2024-11-24T04:48:47,397 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:47,397 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:47,398 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,398 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName b 2024-11-24T04:48:47,399 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:47,399 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:47,399 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,400 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName c 2024-11-24T04:48:47,400 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:47,400 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:47,401 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,401 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,403 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,404 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,404 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,405 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:47,406 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,407 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a212a86e759e7e603b57c25e3c47677; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72154414, jitterRate=0.07518455386161804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:47,407 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:47,408 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a212a86e759e7e603b57c25e3c47677: Running coprocessor pre-open hook at 1732423727383Writing region info on filesystem at 1732423727383Initializing all the Stores at 1732423727384 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423727385 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423727385Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423727385Cleaning up temporary data from old regions at 1732423727404 (+19 ms)Running coprocessor post-open hooks at 1732423727407 (+3 ms)Region opened successfully at 1732423727408 (+1 ms) 2024-11-24T04:48:47,431 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2a212a86e759e7e603b57c25e3c47677 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T04:48:47,443 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:47,983 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:48:48,444 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:49,445 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:50,446 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:51,447 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:52,449 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:52,525 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:48:52,889 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:48:53,450 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:53,702 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T04:48:53,702 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:53,704 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T04:48:53,704 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:53,706 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T04:48:53,706 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T04:48:54,451 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:55,452 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:56,453 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2a212a86e759e7e603b57c25e3c47677/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:48:56,454 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2a212a86e759e7e603b57c25e3c47677: 2024-11-24T04:48:56,454 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:56,468 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2a212a86e759e7e603b57c25e3c47677: 2024-11-24T04:48:56,468 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T04:48:56,468 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2a212a86e759e7e603b57c25e3c47677, disabling compactions & flushes 2024-11-24T04:48:56,468 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:56,468 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:56,468 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. after waiting 0 ms 2024-11-24T04:48:56,468 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:56,468 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:56,469 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. 2024-11-24T04:48:56,469 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2a212a86e759e7e603b57c25e3c47677: Waiting for close lock at 1732423736468Running coprocessor pre-close hooks at 1732423736468Disabling compacts and flushes for region at 1732423736468Disabling writes for close at 1732423736468Writing region close event to WAL at 1732423736468Running coprocessor post-close hooks at 1732423736468Closed at 1732423736469 (+1 ms) 2024-11-24T04:48:56,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741903_1081 (size=4236) 2024-11-24T04:48:56,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741903_1081 (size=4236) 2024-11-24T04:48:56,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741903_1081 (size=4236) 2024-11-24T04:48:56,547 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202, size=4.1 K (4236bytes) 2024-11-24T04:48:56,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 2024-11-24T04:48:56,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 after 0ms 2024-11-24T04:48:56,572 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:56,572 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 took 25ms 2024-11-24T04:48:56,575 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 so closing down 2024-11-24T04:48:56,575 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:56,576 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732423727202.temp 2024-11-24T04:48:56,599 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000004-wal.1732423727202.temp 2024-11-24T04:48:56,599 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:56,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741904_1082 (size=3245) 2024-11-24T04:48:56,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741904_1082 (size=3245) 2024-11-24T04:48:56,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741904_1082 (size=3245) 2024-11-24T04:48:56,606 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000004-wal.1732423727202.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:56,607 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000004-wal.1732423727202.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000026 2024-11-24T04:48:56,608 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-24T04:48:56,608 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202, journal: Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202, size=4.1 K (4236bytes) at 1732423736547Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 so closing down at 1732423736575 (+28 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000004-wal.1732423727202.temp at 1732423736599 (+24 ms)3 split writer threads finished at 1732423736599Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000004-wal.1732423727202.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732423736606 (+7 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000004-wal.1732423727202.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000026 at 1732423736607 (+1 ms)Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732423736608 (+1 ms) 2024-11-24T04:48:56,609 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423727202 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423727202 2024-11-24T04:48:56,610 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000026 2024-11-24T04:48:56,610 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:56,612 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:56,623 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423736612, exclude list is [], retry=0 2024-11-24T04:48:56,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:56,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:56,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:56,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423736612 2024-11-24T04:48:56,654 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:56,654 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a212a86e759e7e603b57c25e3c47677, NAME => 'testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:56,654 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,654 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:56,655 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,655 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,657 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,658 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName a 2024-11-24T04:48:56,658 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:56,658 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:56,659 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,660 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName b 2024-11-24T04:48:56,660 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:56,660 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:56,660 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,661 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a212a86e759e7e603b57c25e3c47677 columnFamilyName c 2024-11-24T04:48:56,661 DEBUG [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:56,662 INFO [StoreOpener-2a212a86e759e7e603b57c25e3c47677-1 {}] regionserver.HStore(327): Store=2a212a86e759e7e603b57c25e3c47677/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:56,662 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,662 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,664 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,664 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000026 2024-11-24T04:48:56,690 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:56,692 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000026 2024-11-24T04:48:56,693 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2a212a86e759e7e603b57c25e3c47677 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T04:48:56,708 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/a/b95f242aca3647708265b10a40290368 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732423736459/Put/seqid=0 2024-11-24T04:48:56,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:48:56,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:48:56,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:48:56,715 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/a/b95f242aca3647708265b10a40290368 2024-11-24T04:48:56,735 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/b/b225aa3237e74e8f95a18f1493d20e21 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732423736455/Put/seqid=0 2024-11-24T04:48:56,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:48:56,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:48:56,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:48:56,742 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/b/b225aa3237e74e8f95a18f1493d20e21 2024-11-24T04:48:56,760 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/c/d958b9cdd3884fcfa5733dcf966e3fcd is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732423736457/Put/seqid=0 2024-11-24T04:48:56,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:48:56,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:48:56,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:48:56,767 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/c/d958b9cdd3884fcfa5733dcf966e3fcd 2024-11-24T04:48:56,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/a/b95f242aca3647708265b10a40290368 as hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/a/b95f242aca3647708265b10a40290368 2024-11-24T04:48:56,778 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/a/b95f242aca3647708265b10a40290368, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T04:48:56,779 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/b/b225aa3237e74e8f95a18f1493d20e21 as hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/b/b225aa3237e74e8f95a18f1493d20e21 2024-11-24T04:48:56,785 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/b/b225aa3237e74e8f95a18f1493d20e21, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T04:48:56,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/.tmp/c/d958b9cdd3884fcfa5733dcf966e3fcd as hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/c/d958b9cdd3884fcfa5733dcf966e3fcd 2024-11-24T04:48:56,792 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/c/d958b9cdd3884fcfa5733dcf966e3fcd, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T04:48:56,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 2a212a86e759e7e603b57c25e3c47677 in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T04:48:56,793 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/0000000000000000026 2024-11-24T04:48:56,794 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,794 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,795 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:56,796 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,798 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsAfterAbortingFlush/2a212a86e759e7e603b57c25e3c47677/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T04:48:56,799 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a212a86e759e7e603b57c25e3c47677; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68237387, jitterRate=0.01681630313396454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:56,799 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a212a86e759e7e603b57c25e3c47677 2024-11-24T04:48:56,799 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a212a86e759e7e603b57c25e3c47677: Running coprocessor pre-open hook at 1732423736655Writing region info on filesystem at 1732423736655Initializing all the Stores at 1732423736656 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423736656Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423736656Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423736656Obtaining lock to block concurrent updates at 1732423736693 (+37 ms)Preparing flush snapshotting stores in 2a212a86e759e7e603b57c25e3c47677 at 1732423736693Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732423736693Flushing stores of testReplayEditsAfterAbortingFlush,,1732423726994.2a212a86e759e7e603b57c25e3c47677. at 1732423736693Flushing 2a212a86e759e7e603b57c25e3c47677/a: creating writer at 1732423736693Flushing 2a212a86e759e7e603b57c25e3c47677/a: appending metadata at 1732423736707 (+14 ms)Flushing 2a212a86e759e7e603b57c25e3c47677/a: closing flushed file at 1732423736707Flushing 2a212a86e759e7e603b57c25e3c47677/b: creating writer at 1732423736720 (+13 ms)Flushing 2a212a86e759e7e603b57c25e3c47677/b: appending metadata at 1732423736734 (+14 ms)Flushing 2a212a86e759e7e603b57c25e3c47677/b: closing flushed file at 1732423736734Flushing 2a212a86e759e7e603b57c25e3c47677/c: creating writer at 1732423736747 (+13 ms)Flushing 2a212a86e759e7e603b57c25e3c47677/c: appending metadata at 1732423736760 (+13 ms)Flushing 2a212a86e759e7e603b57c25e3c47677/c: closing flushed file at 1732423736760Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@519d60f5: reopening flushed file at 1732423736772 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3daefda4: reopening flushed file at 1732423736778 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d5ad3ca: reopening flushed file at 1732423736785 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 2a212a86e759e7e603b57c25e3c47677 in 99ms, sequenceid=26, compaction requested=false; wal=null at 1732423736792 (+7 ms)Cleaning up temporary data from old regions at 1732423736794 (+2 ms)Running coprocessor post-open hooks at 1732423736799 (+5 ms)Region opened successfully at 1732423736799 2024-11-24T04:48:56,818 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418 (was 414) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35100 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35112 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37918 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37912 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:54130 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1122) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=545 (was 607), ProcessCount=11 (was 11), AvailableMemoryMB=10230 (was 10257) 2024-11-24T04:48:56,818 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-24T04:48:56,828 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=545, ProcessCount=11, AvailableMemoryMB=10229 2024-11-24T04:48:56,828 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-24T04:48:56,841 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:56,842 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:56,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:56,845 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26666416, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-26666416, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:56,856 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26666416/hregion-26666416.1732423736846, exclude list is [], retry=0 2024-11-24T04:48:56,858 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:56,858 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:56,859 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:56,881 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26666416/hregion-26666416.1732423736846 2024-11-24T04:48:56,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:56,882 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c39b6ea63fc012e111ad2573f2c662f8, NAME => 'testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:56,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:48:56,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:48:56,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:48:56,890 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:56,891 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,892 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39b6ea63fc012e111ad2573f2c662f8 columnFamilyName a 2024-11-24T04:48:56,892 DEBUG [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:56,893 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(327): Store=c39b6ea63fc012e111ad2573f2c662f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:56,893 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,894 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,894 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,895 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,898 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:56,898 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c39b6ea63fc012e111ad2573f2c662f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74175464, jitterRate=0.10530054569244385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:56,899 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c39b6ea63fc012e111ad2573f2c662f8: Writing region info on filesystem at 1732423736890Initializing all the Stores at 1732423736890Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423736890Cleaning up temporary data from old regions at 1732423736894 (+4 ms)Region opened successfully at 1732423736898 (+4 ms) 2024-11-24T04:48:56,899 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c39b6ea63fc012e111ad2573f2c662f8, disabling compactions & flushes 2024-11-24T04:48:56,899 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,899 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,899 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. after waiting 0 ms 2024-11-24T04:48:56,899 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,899 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,899 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c39b6ea63fc012e111ad2573f2c662f8: Waiting for close lock at 1732423736899Disabling compacts and flushes for region at 1732423736899Disabling writes for close at 1732423736899Writing region close event to WAL at 1732423736899Closed at 1732423736899 2024-11-24T04:48:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741909_1087 (size=175) 2024-11-24T04:48:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741909_1087 (size=175) 2024-11-24T04:48:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741909_1087 (size=175) 2024-11-24T04:48:56,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:56,903 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26666416:(num 1732423736846) 2024-11-24T04:48:56,903 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:56,905 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:56,917 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905, exclude list is [], retry=0 2024-11-24T04:48:56,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:56,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:56,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:56,946 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 2024-11-24T04:48:56,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:56,947 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c39b6ea63fc012e111ad2573f2c662f8, NAME => 'testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:56,947 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:56,947 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,947 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,948 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,949 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39b6ea63fc012e111ad2573f2c662f8 columnFamilyName a 2024-11-24T04:48:56,949 DEBUG [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:56,950 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(327): Store=c39b6ea63fc012e111ad2573f2c662f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:56,950 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,952 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,952 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,952 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,954 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:56,955 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c39b6ea63fc012e111ad2573f2c662f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75090650, jitterRate=0.11893787980079651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:56,955 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c39b6ea63fc012e111ad2573f2c662f8: Writing region info on filesystem at 1732423736947Initializing all the Stores at 1732423736948 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423736948Cleaning up temporary data from old regions at 1732423736952 (+4 ms)Region opened successfully at 1732423736955 (+3 ms) 2024-11-24T04:48:56,963 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c39b6ea63fc012e111ad2573f2c662f8, disabling compactions & flushes 2024-11-24T04:48:56,963 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,964 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,964 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. after waiting 0 ms 2024-11-24T04:48:56,964 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,964 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,964 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. 2024-11-24T04:48:56,964 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c39b6ea63fc012e111ad2573f2c662f8: Waiting for close lock at 1732423736963Disabling compacts and flushes for region at 1732423736963Disabling writes for close at 1732423736964 (+1 ms)Writing region close event to WAL at 1732423736964Closed at 1732423736964 2024-11-24T04:48:56,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741911_1089 (size=1795) 2024-11-24T04:48:56,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741911_1089 (size=1795) 2024-11-24T04:48:56,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741911_1089 (size=1795) 2024-11-24T04:48:56,982 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905, size=1.8 K (1795bytes) 2024-11-24T04:48:56,982 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 2024-11-24T04:48:56,983 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 after 1ms 2024-11-24T04:48:57,027 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:57,027 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 took 45ms 2024-11-24T04:48:57,029 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 so closing down 2024-11-24T04:48:57,029 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:48:57,031 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423736905.temp 2024-11-24T04:48:57,054 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000003-wal.1732423736905.temp 2024-11-24T04:48:57,055 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:48:57,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741912_1090 (size=1795) 2024-11-24T04:48:57,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741912_1090 (size=1795) 2024-11-24T04:48:57,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741912_1090 (size=1795) 2024-11-24T04:48:57,061 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000003-wal.1732423736905.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T04:48:57,063 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000003-wal.1732423736905.temp to hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000012 2024-11-24T04:48:57,063 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-24T04:48:57,063 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905, journal: Splitting hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905, size=1.8 K (1795bytes) at 1732423736982Finishing writing output for hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 so closing down at 1732423737029 (+47 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000003-wal.1732423736905.temp at 1732423737054 (+25 ms)3 split writer threads finished at 1732423737055 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000003-wal.1732423736905.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732423737061 (+6 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000003-wal.1732423736905.temp to hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000012 at 1732423737063 (+2 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732423737063 2024-11-24T04:48:57,065 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423736905 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423736905 2024-11-24T04:48:57,065 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000012 2024-11-24T04:48:57,093 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:57,460 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:57,462 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:57,474 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423737462, exclude list is [], retry=0 2024-11-24T04:48:57,477 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:57,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:57,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:57,504 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423737462 2024-11-24T04:48:57,504 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:57,504 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c39b6ea63fc012e111ad2573f2c662f8, NAME => 'testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:57,504 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:57,504 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,504 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,508 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,511 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39b6ea63fc012e111ad2573f2c662f8 columnFamilyName a 2024-11-24T04:48:57,511 DEBUG [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,512 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(327): Store=c39b6ea63fc012e111ad2573f2c662f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,512 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,515 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,516 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000012 2024-11-24T04:48:57,541 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:48:57,541 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000012 2024-11-24T04:48:57,542 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c39b6ea63fc012e111ad2573f2c662f8 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T04:48:57,557 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/.tmp/a/4f2ca11dcf974ca0bb0c12d5b05f6901 is 79, key is testDatalossWhenInputError/a:x0/1732423736955/Put/seqid=0 2024-11-24T04:48:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:48:57,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:48:57,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:48:57,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/.tmp/a/4f2ca11dcf974ca0bb0c12d5b05f6901 2024-11-24T04:48:57,577 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/.tmp/a/4f2ca11dcf974ca0bb0c12d5b05f6901 as hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/a/4f2ca11dcf974ca0bb0c12d5b05f6901 2024-11-24T04:48:57,586 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/a/4f2ca11dcf974ca0bb0c12d5b05f6901, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T04:48:57,586 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c39b6ea63fc012e111ad2573f2c662f8 in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T04:48:57,586 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/0000000000000000012 2024-11-24T04:48:57,588 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,588 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,591 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,593 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T04:48:57,594 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c39b6ea63fc012e111ad2573f2c662f8; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73379234, jitterRate=0.09343579411506653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:57,595 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c39b6ea63fc012e111ad2573f2c662f8: Writing region info on filesystem at 1732423737504Initializing all the Stores at 1732423737508 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737508Obtaining lock to block concurrent updates at 1732423737542 (+34 ms)Preparing flush snapshotting stores in c39b6ea63fc012e111ad2573f2c662f8 at 1732423737542Finished memstore snapshotting testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732423737542Flushing stores of testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8. at 1732423737542Flushing c39b6ea63fc012e111ad2573f2c662f8/a: creating writer at 1732423737542Flushing c39b6ea63fc012e111ad2573f2c662f8/a: appending metadata at 1732423737557 (+15 ms)Flushing c39b6ea63fc012e111ad2573f2c662f8/a: closing flushed file at 1732423737557Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@192a04ff: reopening flushed file at 1732423737575 (+18 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c39b6ea63fc012e111ad2573f2c662f8 in 44ms, sequenceid=12, compaction requested=false; wal=null at 1732423737586 (+11 ms)Cleaning up temporary data from old regions at 1732423737588 (+2 ms)Region opened successfully at 1732423737595 (+7 ms) 2024-11-24T04:48:57,598 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c39b6ea63fc012e111ad2573f2c662f8, NAME => 'testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:57,598 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423736841.c39b6ea63fc012e111ad2573f2c662f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:57,598 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,598 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,600 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,601 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c39b6ea63fc012e111ad2573f2c662f8 columnFamilyName a 2024-11-24T04:48:57,601 DEBUG [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,608 DEBUG [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/a/4f2ca11dcf974ca0bb0c12d5b05f6901 2024-11-24T04:48:57,608 INFO [StoreOpener-c39b6ea63fc012e111ad2573f2c662f8-1 {}] regionserver.HStore(327): Store=c39b6ea63fc012e111ad2573f2c662f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,608 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,609 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,610 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,611 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,611 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,613 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c39b6ea63fc012e111ad2573f2c662f8 2024-11-24T04:48:57,615 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testDatalossWhenInputError/c39b6ea63fc012e111ad2573f2c662f8/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T04:48:57,616 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c39b6ea63fc012e111ad2573f2c662f8; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69590611, jitterRate=0.03698091208934784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:48:57,616 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c39b6ea63fc012e111ad2573f2c662f8: Writing region info on filesystem at 1732423737599Initializing all the Stores at 1732423737600 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737600Cleaning up temporary data from old regions at 1732423737611 (+11 ms)Region opened successfully at 1732423737616 (+5 ms) 2024-11-24T04:48:57,634 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37996 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35100 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:37918 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:54198 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2110872418-172.17.0.2-1732423698697:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35158 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=545 (was 545), ProcessCount=11 (was 11), AvailableMemoryMB=10205 (was 10229) 2024-11-24T04:48:57,634 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T04:48:57,647 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=545, ProcessCount=11, AvailableMemoryMB=10205 2024-11-24T04:48:57,648 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T04:48:57,667 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:57,669 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:48:57,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:48:57,673 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63233314, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-63233314, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:57,685 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63233314/hregion-63233314.1732423737674, exclude list is [], retry=0 2024-11-24T04:48:57,687 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:57,687 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:57,688 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:57,712 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63233314/hregion-63233314.1732423737674 2024-11-24T04:48:57,712 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:48:57,712 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f86d12a14f9dbc84848af6f664c644c1, NAME => 'testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:48:57,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:48:57,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:48:57,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:48:57,724 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:57,725 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,727 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName a 2024-11-24T04:48:57,727 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,727 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,727 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,728 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName b 2024-11-24T04:48:57,728 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,729 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,729 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,730 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName c 2024-11-24T04:48:57,730 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,730 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,731 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,731 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,733 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,733 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,733 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:57,735 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,737 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:48:57,737 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f86d12a14f9dbc84848af6f664c644c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72292425, jitterRate=0.07724107801914215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:57,737 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f86d12a14f9dbc84848af6f664c644c1: Writing region info on filesystem at 1732423737724Initializing all the Stores at 1732423737725 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737725Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737725Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737725Cleaning up temporary data from old regions at 1732423737733 (+8 ms)Region opened successfully at 1732423737737 (+4 ms) 2024-11-24T04:48:57,738 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f86d12a14f9dbc84848af6f664c644c1, disabling compactions & flushes 2024-11-24T04:48:57,738 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:48:57,738 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:48:57,738 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. after waiting 0 ms 2024-11-24T04:48:57,738 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:48:57,738 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:48:57,738 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f86d12a14f9dbc84848af6f664c644c1: Waiting for close lock at 1732423737738Disabling compacts and flushes for region at 1732423737738Disabling writes for close at 1732423737738Writing region close event to WAL at 1732423737738Closed at 1732423737738 2024-11-24T04:48:57,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741915_1093 (size=175) 2024-11-24T04:48:57,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741915_1093 (size=175) 2024-11-24T04:48:57,741 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-63233314/hregion-63233314.1732423737674 not finished, retry = 0 2024-11-24T04:48:57,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741915_1093 (size=175) 2024-11-24T04:48:57,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:48:57,844 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63233314:(num 1732423737674) 2024-11-24T04:48:57,845 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:48:57,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:48:57,862 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848, exclude list is [], retry=0 2024-11-24T04:48:57,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:48:57,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:48:57,866 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:48:57,903 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 2024-11-24T04:48:57,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:48:57,903 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f86d12a14f9dbc84848af6f664c644c1, NAME => 'testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:48:57,904 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:48:57,904 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,904 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,910 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,912 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName a 2024-11-24T04:48:57,912 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,912 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,912 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,913 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName b 2024-11-24T04:48:57,914 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,914 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,914 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,915 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName c 2024-11-24T04:48:57,915 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:48:57,916 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:48:57,916 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,917 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,918 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,919 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,919 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,920 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:48:57,925 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:48:57,926 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f86d12a14f9dbc84848af6f664c644c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71005214, jitterRate=0.058060139417648315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:48:57,926 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f86d12a14f9dbc84848af6f664c644c1: Writing region info on filesystem at 1732423737904Initializing all the Stores at 1732423737906 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737906Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737908 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423737908Cleaning up temporary data from old regions at 1732423737919 (+11 ms)Region opened successfully at 1732423737926 (+7 ms) 2024-11-24T04:48:57,934 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732423737934/Put/seqid=0 2024-11-24T04:48:57,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:48:57,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:48:57,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:48:57,958 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732423737957/Put/seqid=0 2024-11-24T04:48:57,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:48:57,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:48:57,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:48:58,395 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732423738395/Put/seqid=0 2024-11-24T04:48:58,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:48:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:48:58,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:48:58,406 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in f86d12a14f9dbc84848af6f664c644c1/a 2024-11-24T04:48:58,410 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T04:48:58,410 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:48:58,410 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in f86d12a14f9dbc84848af6f664c644c1/a 2024-11-24T04:48:58,415 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T04:48:58,415 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:48:58,415 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in f86d12a14f9dbc84848af6f664c644c1/a 2024-11-24T04:48:58,422 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T04:48:58,422 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:48:58,422 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f86d12a14f9dbc84848af6f664c644c1 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T04:48:58,443 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/.tmp/a/fcaa21450538435d8f1e5115bcc1317e is 55, key is testCompactedBulkLoadedFiles/a:a/1732423737927/Put/seqid=0 2024-11-24T04:48:58,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:48:58,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:48:58,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:48:58,453 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/.tmp/a/fcaa21450538435d8f1e5115bcc1317e 2024-11-24T04:48:58,460 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/.tmp/a/fcaa21450538435d8f1e5115bcc1317e as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e 2024-11-24T04:48:58,467 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T04:48:58,468 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for f86d12a14f9dbc84848af6f664c644c1 in 46ms, sequenceid=4, compaction requested=false 2024-11-24T04:48:58,468 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f86d12a14f9dbc84848af6f664c644c1: 2024-11-24T04:48:58,469 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ 2024-11-24T04:48:58,470 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ 2024-11-24T04:48:58,472 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ 2024-11-24T04:48:58,473 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile0 into f86d12a14f9dbc84848af6f664c644c1/a as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ - updating store file list. 2024-11-24T04:48:58,478 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5c84cfa402094daea153c620521e8b1b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:48:58,478 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ into f86d12a14f9dbc84848af6f664c644c1/a 2024-11-24T04:48:58,478 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile0 into f86d12a14f9dbc84848af6f664c644c1/a (new location: hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_) 2024-11-24T04:48:58,479 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile1 into f86d12a14f9dbc84848af6f664c644c1/a as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ - updating store file list. 2024-11-24T04:48:58,484 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ae012beffee1462f99807962f2517b83_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:48:58,485 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ into f86d12a14f9dbc84848af6f664c644c1/a 2024-11-24T04:48:58,485 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile1 into f86d12a14f9dbc84848af6f664c644c1/a (new location: hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_) 2024-11-24T04:48:58,488 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile2 into f86d12a14f9dbc84848af6f664c644c1/a as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ - updating store file list. 2024-11-24T04:48:58,492 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9388e20a5fc5432b83de0020199a962f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:48:58,492 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ into f86d12a14f9dbc84848af6f664c644c1/a 2024-11-24T04:48:58,492 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43985/hbase/testCompactedBulkLoadedFiles/hfile2 into f86d12a14f9dbc84848af6f664c644c1/a (new location: hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_) 2024-11-24T04:48:58,501 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T04:48:58,501 DEBUG [Time-limited test {}] regionserver.HStore(1541): f86d12a14f9dbc84848af6f664c644c1/a is initiating major compaction (all files) 2024-11-24T04:48:58,501 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of f86d12a14f9dbc84848af6f664c644c1/a in testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:48:58,501 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_] into tmpdir=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/.tmp, totalSize=19.3 K 2024-11-24T04:48:58,502 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fcaa21450538435d8f1e5115bcc1317e, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732423737927 2024-11-24T04:48:58,503 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5c84cfa402094daea153c620521e8b1b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:48:58,503 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ae012beffee1462f99807962f2517b83_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:48:58,503 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9388e20a5fc5432b83de0020199a962f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:48:58,519 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/.tmp/a/27ddbd34911245c1a987985d265838ba is 55, key is testCompactedBulkLoadedFiles/a:a/1732423737927/Put/seqid=0 2024-11-24T04:48:58,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:48:58,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:48:58,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:48:58,545 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/.tmp/a/27ddbd34911245c1a987985d265838ba as hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/27ddbd34911245c1a987985d265838ba 2024-11-24T04:48:58,552 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in f86d12a14f9dbc84848af6f664c644c1/a of f86d12a14f9dbc84848af6f664c644c1 into 27ddbd34911245c1a987985d265838ba(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:48:58,552 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for f86d12a14f9dbc84848af6f664c644c1: 2024-11-24T04:48:58,552 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T04:48:58,552 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T04:48:58,597 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848, size=0 (0bytes) 2024-11-24T04:48:58,597 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 might be still open, length is 0 2024-11-24T04:48:58,598 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 2024-11-24T04:48:58,598 WARN [IPC Server handler 1 on default port 43985 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-24T04:48:58,599 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 after 0ms 2024-11-24T04:49:01,496 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:54238 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45441:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54238 dst: /127.0.0.1:45441 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45441 remote=/127.0.0.1:54238]. Total timeout mills is 60000, 57055 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:01,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:35178 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43379:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35178 dst: /127.0.0.1:43379 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:01,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:38032 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38032 dst: /127.0.0.1:35959 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:01,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741917_1101 (size=1794) 2024-11-24T04:49:01,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741917_1101 (size=1794) 2024-11-24T04:49:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741917_1101 (size=1794) 2024-11-24T04:49:01,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:49:02,599 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 after 4001ms 2024-11-24T04:49:02,630 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:49:02,630 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 took 4033ms 2024-11-24T04:49:02,632 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848; continuing. 2024-11-24T04:49:02,632 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 so closing down 2024-11-24T04:49:02,632 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:49:02,633 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423737848.temp 2024-11-24T04:49:02,662 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000003-wal.1732423737848.temp 2024-11-24T04:49:02,662 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:49:02,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741923_1102 (size=800) 2024-11-24T04:49:02,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741923_1102 (size=800) 2024-11-24T04:49:02,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741923_1102 (size=800) 2024-11-24T04:49:03,085 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000003-wal.1732423737848.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:49:03,086 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000003-wal.1732423737848.temp to hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000008 2024-11-24T04:49:03,086 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 456 ms; skipped=3; WAL=hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:49:03,086 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848, journal: Splitting hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848, size=0 (0bytes) at 1732423738597Finishing writing output for hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 so closing down at 1732423742632 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000003-wal.1732423737848.temp at 1732423742662 (+30 ms)3 split writer threads finished at 1732423742662Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000003-wal.1732423737848.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732423743085 (+423 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000003-wal.1732423737848.temp to hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000008 at 1732423743086 (+1 ms)Processed 5 edits across 1 Regions in 456 ms; skipped=3; WAL=hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848, size=0, length=0, corrupted=false, cancelled=false at 1732423743086 2024-11-24T04:49:03,088 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423737848 2024-11-24T04:49:03,089 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000008 2024-11-24T04:49:03,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:49:03,091 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:49:03,108 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423743091, exclude list is [], retry=0 2024-11-24T04:49:03,110 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:49:03,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:49:03,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:49:03,147 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423743091 2024-11-24T04:49:03,147 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:49:03,147 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f86d12a14f9dbc84848af6f664c644c1, NAME => 'testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:49:03,147 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:49:03,148 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,148 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,150 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,151 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName a 2024-11-24T04:49:03,151 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,157 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/27ddbd34911245c1a987985d265838ba 2024-11-24T04:49:03,162 DEBUG [StoreFileOpener-f86d12a14f9dbc84848af6f664c644c1-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5c84cfa402094daea153c620521e8b1b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:49:03,162 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ 2024-11-24T04:49:03,166 DEBUG [StoreFileOpener-f86d12a14f9dbc84848af6f664c644c1-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9388e20a5fc5432b83de0020199a962f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:49:03,166 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ 2024-11-24T04:49:03,170 DEBUG [StoreFileOpener-f86d12a14f9dbc84848af6f664c644c1-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ae012beffee1462f99807962f2517b83_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:49:03,170 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ 2024-11-24T04:49:03,174 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e 2024-11-24T04:49:03,174 WARN [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@14056e20 2024-11-24T04:49:03,174 WARN [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@14056e20 2024-11-24T04:49:03,174 WARN [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@14056e20 2024-11-24T04:49:03,174 WARN [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@14056e20 2024-11-24T04:49:03,174 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e] to archive 2024-11-24T04:49:03,175 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:49:03,177 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ to hdfs://localhost:43985/hbase/archive/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/5c84cfa402094daea153c620521e8b1b_SeqId_4_ 2024-11-24T04:49:03,179 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ to hdfs://localhost:43985/hbase/archive/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/9388e20a5fc5432b83de0020199a962f_SeqId_4_ 2024-11-24T04:49:03,180 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ to hdfs://localhost:43985/hbase/archive/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/ae012beffee1462f99807962f2517b83_SeqId_4_ 2024-11-24T04:49:03,181 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e to hdfs://localhost:43985/hbase/archive/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/a/fcaa21450538435d8f1e5115bcc1317e 2024-11-24T04:49:03,181 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,181 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,182 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName b 2024-11-24T04:49:03,182 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,182 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,182 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,183 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f86d12a14f9dbc84848af6f664c644c1 columnFamilyName c 2024-11-24T04:49:03,183 DEBUG [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,184 INFO [StoreOpener-f86d12a14f9dbc84848af6f664c644c1-1 {}] regionserver.HStore(327): Store=f86d12a14f9dbc84848af6f664c644c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,184 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,185 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,187 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000008 2024-11-24T04:49:03,223 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:49:03,226 DEBUG [Time-limited test {}] regionserver.HRegion(5836): f86d12a14f9dbc84848af6f664c644c1 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "f86d12a14f9dbc84848af6f664c644c1" family_name: "a" compaction_input: "fcaa21450538435d8f1e5115bcc1317e" compaction_input: "5c84cfa402094daea153c620521e8b1b_SeqId_4_" compaction_input: "ae012beffee1462f99807962f2517b83_SeqId_4_" compaction_input: "9388e20a5fc5432b83de0020199a962f_SeqId_4_" compaction_output: "27ddbd34911245c1a987985d265838ba" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T04:49:03,226 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T04:49:03,226 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000008 2024-11-24T04:49:03,227 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/0000000000000000008 2024-11-24T04:49:03,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,228 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,229 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:49:03,231 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f86d12a14f9dbc84848af6f664c644c1 2024-11-24T04:49:03,233 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testCompactedBulkLoadedFiles/f86d12a14f9dbc84848af6f664c644c1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T04:49:03,234 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f86d12a14f9dbc84848af6f664c644c1; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64468743, jitterRate=-0.03934086859226227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:49:03,234 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f86d12a14f9dbc84848af6f664c644c1: Writing region info on filesystem at 1732423743148Initializing all the Stores at 1732423743150 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743150Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743150Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743150Cleaning up temporary data from old regions at 1732423743228 (+78 ms)Region opened successfully at 1732423743234 (+6 ms) 2024-11-24T04:49:03,237 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f86d12a14f9dbc84848af6f664c644c1, disabling compactions & flushes 2024-11-24T04:49:03,237 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:49:03,237 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:49:03,237 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. after waiting 0 ms 2024-11-24T04:49:03,237 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:49:03,238 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732423737667.f86d12a14f9dbc84848af6f664c644c1. 2024-11-24T04:49:03,238 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f86d12a14f9dbc84848af6f664c644c1: Waiting for close lock at 1732423743237Disabling compacts and flushes for region at 1732423743237Disabling writes for close at 1732423743237Writing region close event to WAL at 1732423743238 (+1 ms)Closed at 1732423743238 2024-11-24T04:49:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741924_1103 (size=175) 2024-11-24T04:49:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741924_1103 (size=175) 2024-11-24T04:49:03,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741924_1103 (size=175) 2024-11-24T04:49:03,243 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:49:03,243 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423743091) 2024-11-24T04:49:03,261 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=439 (was 428) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1401180430_22 at /127.0.0.1:51952 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43985 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1401180430_22 at /127.0.0.1:43122 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1401180430_22 at /127.0.0.1:59174 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:43985 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=534 (was 545), ProcessCount=11 (was 11), AvailableMemoryMB=10197 (was 10205) 2024-11-24T04:49:03,261 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T04:49:03,278 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=534, ProcessCount=11, AvailableMemoryMB=10197 2024-11-24T04:49:03,278 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T04:49:03,297 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:49:03,299 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:49:03,300 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:49:03,303 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24994067, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/hregion-24994067, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:49:03,318 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24994067/hregion-24994067.1732423743303, exclude list is [], retry=0 2024-11-24T04:49:03,321 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:49:03,323 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:49:03,323 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:49:03,352 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24994067/hregion-24994067.1732423743303 2024-11-24T04:49:03,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:49:03,353 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d9a2369468b78e2bdc86c3499284cfae, NAME => 'testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43985/hbase 2024-11-24T04:49:03,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:49:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:49:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:49:03,376 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:49:03,377 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,379 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName a 2024-11-24T04:49:03,379 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,380 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,380 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,381 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName b 2024-11-24T04:49:03,381 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,382 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,382 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,383 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName c 2024-11-24T04:49:03,384 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,384 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,384 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,385 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,386 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,386 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,387 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:49:03,389 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,393 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:49:03,393 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9a2369468b78e2bdc86c3499284cfae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75304919, jitterRate=0.12213073670864105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:49:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9a2369468b78e2bdc86c3499284cfae: Writing region info on filesystem at 1732423743376Initializing all the Stores at 1732423743377 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743377Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743377Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743377Cleaning up temporary data from old regions at 1732423743386 (+9 ms)Region opened successfully at 1732423743393 (+7 ms) 2024-11-24T04:49:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9a2369468b78e2bdc86c3499284cfae, disabling compactions & flushes 2024-11-24T04:49:03,393 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. after waiting 0 ms 2024-11-24T04:49:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,400 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9a2369468b78e2bdc86c3499284cfae: Waiting for close lock at 1732423743393Disabling compacts and flushes for region at 1732423743393Disabling writes for close at 1732423743394 (+1 ms)Writing region close event to WAL at 1732423743400 (+6 ms)Closed at 1732423743400 2024-11-24T04:49:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741925_1104 (size=175) 2024-11-24T04:49:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741925_1104 (size=175) 2024-11-24T04:49:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741925_1104 (size=175) 2024-11-24T04:49:03,421 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:49:03,421 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24994067:(num 1732423743303) 2024-11-24T04:49:03,421 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:49:03,425 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:49:03,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426, exclude list is [], retry=0 2024-11-24T04:49:03,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:49:03,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:49:03,446 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:49:03,482 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 2024-11-24T04:49:03,482 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:44569:44569)] 2024-11-24T04:49:03,482 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9a2369468b78e2bdc86c3499284cfae, NAME => 'testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:49:03,483 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:49:03,483 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,483 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,484 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,485 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName a 2024-11-24T04:49:03,485 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,486 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,486 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,487 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName b 2024-11-24T04:49:03,487 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,487 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,487 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,488 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName c 2024-11-24T04:49:03,488 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,489 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,489 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,493 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,494 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:49:03,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,496 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9a2369468b78e2bdc86c3499284cfae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64584829, jitterRate=-0.03761105239391327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:49:03,496 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9a2369468b78e2bdc86c3499284cfae: Writing region info on filesystem at 1732423743483Initializing all the Stores at 1732423743484 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743484Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743484Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743484Cleaning up temporary data from old regions at 1732423743493 (+9 ms)Region opened successfully at 1732423743496 (+3 ms) 2024-11-24T04:49:03,506 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9a2369468b78e2bdc86c3499284cfae 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T04:49:03,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/a/5464e9ae51054cb59899fa39cfdb1212 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732423743496/Put/seqid=0 2024-11-24T04:49:03,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:49:03,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:49:03,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:49:03,547 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/a/5464e9ae51054cb59899fa39cfdb1212 2024-11-24T04:49:03,554 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/a/5464e9ae51054cb59899fa39cfdb1212 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/a/5464e9ae51054cb59899fa39cfdb1212 2024-11-24T04:49:03,560 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/a/5464e9ae51054cb59899fa39cfdb1212, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T04:49:03,561 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for d9a2369468b78e2bdc86c3499284cfae in 55ms, sequenceid=13, compaction requested=false 2024-11-24T04:49:03,561 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d9a2369468b78e2bdc86c3499284cfae: 2024-11-24T04:49:03,583 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9a2369468b78e2bdc86c3499284cfae, disabling compactions & flushes 2024-11-24T04:49:03,583 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,584 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,584 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. after waiting 0 ms 2024-11-24T04:49:03,584 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,585 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,585 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:03,585 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9a2369468b78e2bdc86c3499284cfae: Waiting for close lock at 1732423743583Disabling compacts and flushes for region at 1732423743583Disabling writes for close at 1732423743584 (+1 ms)Writing region close event to WAL at 1732423743585 (+1 ms)Closed at 1732423743585 2024-11-24T04:49:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741927_1106 (size=6243) 2024-11-24T04:49:03,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741927_1106 (size=6243) 2024-11-24T04:49:03,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741927_1106 (size=6243) 2024-11-24T04:49:03,613 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426, size=6.1 K (6243bytes) 2024-11-24T04:49:03,613 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 2024-11-24T04:49:03,614 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 after 1ms 2024-11-24T04:49:03,641 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:49:03,641 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 took 28ms 2024-11-24T04:49:03,643 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 so closing down 2024-11-24T04:49:03,643 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:49:03,644 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423743426.temp 2024-11-24T04:49:03,672 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000003-wal.1732423743426.temp 2024-11-24T04:49:03,672 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:49:03,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741929_1108 (size=5575) 2024-11-24T04:49:03,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741929_1108 (size=5575) 2024-11-24T04:49:03,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741929_1108 (size=5575) 2024-11-24T04:49:03,687 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000003-wal.1732423743426.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:49:03,689 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000003-wal.1732423743426.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000035 2024-11-24T04:49:03,689 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 47 ms; skipped=2; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-24T04:49:03,689 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426, journal: Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426, size=6.1 K (6243bytes) at 1732423743613Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 so closing down at 1732423743643 (+30 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000003-wal.1732423743426.temp at 1732423743672 (+29 ms)3 split writer threads finished at 1732423743672Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000003-wal.1732423743426.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732423743687 (+15 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000003-wal.1732423743426.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000035 at 1732423743689 (+2 ms)Processed 32 edits across 1 Regions in 47 ms; skipped=2; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732423743689 2024-11-24T04:49:03,691 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743426 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423743426 2024-11-24T04:49:03,692 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000035 2024-11-24T04:49:03,692 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:49:03,694 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:49:03,702 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T04:49:03,702 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T04:49:03,702 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T04:49:03,702 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T04:49:03,709 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694, exclude list is [], retry=0 2024-11-24T04:49:03,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:49:03,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:49:03,712 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:49:03,739 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 2024-11-24T04:49:03,740 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45051:45051),(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929)] 2024-11-24T04:49:03,740 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9a2369468b78e2bdc86c3499284cfae, NAME => 'testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:49:03,740 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:49:03,740 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,740 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,741 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,742 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName a 2024-11-24T04:49:03,742 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,747 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/a/5464e9ae51054cb59899fa39cfdb1212 2024-11-24T04:49:03,747 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,747 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,748 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName b 2024-11-24T04:49:03,748 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,749 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,749 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,749 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName c 2024-11-24T04:49:03,749 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:03,750 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:03,750 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,753 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,754 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,754 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000035 2024-11-24T04:49:03,784 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:49:03,785 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000035 2024-11-24T04:49:03,785 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9a2369468b78e2bdc86c3499284cfae 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T04:49:03,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/b/62956f261a5343c29eff68db7443588b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732423743561/Put/seqid=0 2024-11-24T04:49:03,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:49:03,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:49:03,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:49:03,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/b/62956f261a5343c29eff68db7443588b 2024-11-24T04:49:03,841 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/c/9cdeedfc4f154149ba60ac9c8debc3dc is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732423743570/Put/seqid=0 2024-11-24T04:49:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:49:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:49:03,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:49:03,856 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/c/9cdeedfc4f154149ba60ac9c8debc3dc 2024-11-24T04:49:03,863 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/b/62956f261a5343c29eff68db7443588b as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/b/62956f261a5343c29eff68db7443588b 2024-11-24T04:49:03,869 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/b/62956f261a5343c29eff68db7443588b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T04:49:03,870 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/c/9cdeedfc4f154149ba60ac9c8debc3dc as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/c/9cdeedfc4f154149ba60ac9c8debc3dc 2024-11-24T04:49:03,878 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/c/9cdeedfc4f154149ba60ac9c8debc3dc, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T04:49:03,878 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d9a2369468b78e2bdc86c3499284cfae in 93ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T04:49:03,879 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000035 2024-11-24T04:49:03,880 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,881 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,881 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:49:03,883 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:03,887 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T04:49:03,888 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9a2369468b78e2bdc86c3499284cfae; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63037926, jitterRate=-0.06066170334815979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:49:03,889 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9a2369468b78e2bdc86c3499284cfae: Writing region info on filesystem at 1732423743740Initializing all the Stores at 1732423743741 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743741Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743741Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423743741Obtaining lock to block concurrent updates at 1732423743785 (+44 ms)Preparing flush snapshotting stores in d9a2369468b78e2bdc86c3499284cfae at 1732423743785Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732423743785Flushing stores of testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. at 1732423743785Flushing d9a2369468b78e2bdc86c3499284cfae/b: creating writer at 1732423743786 (+1 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/b: appending metadata at 1732423743804 (+18 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/b: closing flushed file at 1732423743804Flushing d9a2369468b78e2bdc86c3499284cfae/c: creating writer at 1732423743826 (+22 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/c: appending metadata at 1732423743840 (+14 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/c: closing flushed file at 1732423743840Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a4ecd5: reopening flushed file at 1732423743862 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@319ba63d: reopening flushed file at 1732423743869 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d9a2369468b78e2bdc86c3499284cfae in 93ms, sequenceid=35, compaction requested=false; wal=null at 1732423743878 (+9 ms)Cleaning up temporary data from old regions at 1732423743881 (+3 ms)Region opened successfully at 1732423743889 (+8 ms) 2024-11-24T04:49:03,975 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694, size=0 (0bytes) 2024-11-24T04:49:03,975 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 might be still open, length is 0 2024-11-24T04:49:03,975 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 2024-11-24T04:49:03,975 WARN [IPC Server handler 4 on default port 43985 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-24T04:49:03,975 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 after 0ms 2024-11-24T04:49:04,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:59232 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45441:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59232 dst: /127.0.0.1:45441 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45441 remote=/127.0.0.1:59232]. Total timeout mills is 60000, 59426 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:04,498 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:51996 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43379:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51996 dst: /127.0.0.1:43379 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43379 remote=/127.0.0.1:51996]. Total timeout mills is 60000, 59424 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:04,499 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-680574894_22 at /127.0.0.1:43162 [Receiving block BP-2110872418-172.17.0.2-1732423698697:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35959:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43162 dst: /127.0.0.1:35959 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35959 remote=/127.0.0.1:43162]. Total timeout mills is 60000, 59423 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:04,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741930_1112 (size=5567) 2024-11-24T04:49:04,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741930_1112 (size=5567) 2024-11-24T04:49:04,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741930_1112 (size=5567) 2024-11-24T04:49:06,818 INFO [master/4464c5b832df:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T04:49:06,818 INFO [master/4464c5b832df:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T04:49:07,976 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 after 4001ms 2024-11-24T04:49:08,004 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:49:08,004 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 took 4030ms 2024-11-24T04:49:08,006 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694; continuing. 2024-11-24T04:49:08,006 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 so closing down 2024-11-24T04:49:08,006 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:49:08,007 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732423743694.temp 2024-11-24T04:49:08,033 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000037-wal.1732423743694.temp 2024-11-24T04:49:08,033 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:49:08,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741933_1113 (size=5575) 2024-11-24T04:49:08,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741933_1113 (size=5575) 2024-11-24T04:49:08,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741933_1113 (size=5575) 2024-11-24T04:49:08,043 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000037-wal.1732423743694.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:49:08,045 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000037-wal.1732423743694.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000066 2024-11-24T04:49:08,045 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:49:08,045 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694, journal: Splitting hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694, size=0 (0bytes) at 1732423743975Finishing writing output for hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 so closing down at 1732423748006 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000037-wal.1732423743694.temp at 1732423748033 (+27 ms)3 split writer threads finished at 1732423748033Closed recovered edits writer path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000037-wal.1732423743694.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732423748043 (+10 ms)Rename recovered edits hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000037-wal.1732423743694.temp to hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000066 at 1732423748045 (+2 ms)Processed 30 edits across 1 Regions in 41 ms; skipped=0; WAL=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694, size=0, length=0, corrupted=false, cancelled=false at 1732423748045 2024-11-24T04:49:08,048 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 to hdfs://localhost:43985/hbase/oldWALs/wal.1732423743694 2024-11-24T04:49:08,048 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000066 2024-11-24T04:49:08,049 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:49:08,052 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43985/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297, archiveDir=hdfs://localhost:43985/hbase/oldWALs, maxLogs=32 2024-11-24T04:49:08,070 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423748052, exclude list is [], retry=0 2024-11-24T04:49:08,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43379,DS-859f7789-1b26-4152-ac45-6d8c026b65ca,DISK] 2024-11-24T04:49:08,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35959,DS-cd2cc379-b259-4d06-9e58-ecb813f8e8ad,DISK] 2024-11-24T04:49:08,073 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45441,DS-e5725c27-fd79-47c0-a49e-0cea0536f84b,DISK] 2024-11-24T04:49:08,112 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423748052 2024-11-24T04:49:08,112 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44569:44569),(127.0.0.1/127.0.0.1:44929:44929),(127.0.0.1/127.0.0.1:45051:45051)] 2024-11-24T04:49:08,113 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:49:08,114 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,115 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName a 2024-11-24T04:49:08,115 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:08,122 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/a/5464e9ae51054cb59899fa39cfdb1212 2024-11-24T04:49:08,122 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:08,122 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,123 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName b 2024-11-24T04:49:08,123 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:08,128 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/b/62956f261a5343c29eff68db7443588b 2024-11-24T04:49:08,128 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:08,128 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,129 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a2369468b78e2bdc86c3499284cfae columnFamilyName c 2024-11-24T04:49:08,130 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:49:08,135 DEBUG [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/c/9cdeedfc4f154149ba60ac9c8debc3dc 2024-11-24T04:49:08,135 INFO [StoreOpener-d9a2369468b78e2bdc86c3499284cfae-1 {}] regionserver.HStore(327): Store=d9a2369468b78e2bdc86c3499284cfae/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:49:08,136 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,137 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,138 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,139 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000066 2024-11-24T04:49:08,178 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T04:49:08,182 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000066 2024-11-24T04:49:08,182 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9a2369468b78e2bdc86c3499284cfae 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T04:49:08,206 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/a/9984f4e7da7344178be4a7bb968bc81b is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732423743898/Put/seqid=0 2024-11-24T04:49:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:49:08,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:49:08,225 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/a/9984f4e7da7344178be4a7bb968bc81b 2024-11-24T04:49:08,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:49:08,266 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/b/ef1a6583f2d74751a039e038c5c79eba is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732423743907/Put/seqid=0 2024-11-24T04:49:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:49:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:49:08,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:49:08,275 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/b/ef1a6583f2d74751a039e038c5c79eba 2024-11-24T04:49:08,298 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/c/41ecd52b88e64f6ebcb8f55b48f0b084 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732423743916/Put/seqid=0 2024-11-24T04:49:08,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:49:08,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:49:08,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:49:08,316 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/c/41ecd52b88e64f6ebcb8f55b48f0b084 2024-11-24T04:49:08,323 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/a/9984f4e7da7344178be4a7bb968bc81b as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/a/9984f4e7da7344178be4a7bb968bc81b 2024-11-24T04:49:08,333 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/a/9984f4e7da7344178be4a7bb968bc81b, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:49:08,334 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/b/ef1a6583f2d74751a039e038c5c79eba as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/b/ef1a6583f2d74751a039e038c5c79eba 2024-11-24T04:49:08,340 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/b/ef1a6583f2d74751a039e038c5c79eba, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:49:08,341 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/.tmp/c/41ecd52b88e64f6ebcb8f55b48f0b084 as hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/c/41ecd52b88e64f6ebcb8f55b48f0b084 2024-11-24T04:49:08,348 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/c/41ecd52b88e64f6ebcb8f55b48f0b084, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:49:08,349 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d9a2369468b78e2bdc86c3499284cfae in 167ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T04:49:08,349 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/0000000000000000066 2024-11-24T04:49:08,351 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,351 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,351 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:49:08,353 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9a2369468b78e2bdc86c3499284cfae 2024-11-24T04:49:08,355 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/hbase/data/default/testReplayEditsWrittenViaHRegion/d9a2369468b78e2bdc86c3499284cfae/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T04:49:08,356 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9a2369468b78e2bdc86c3499284cfae; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60319866, jitterRate=-0.10116395354270935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:49:08,357 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9a2369468b78e2bdc86c3499284cfae: Writing region info on filesystem at 1732423748113Initializing all the Stores at 1732423748114 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423748114Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423748114Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423748114Obtaining lock to block concurrent updates at 1732423748182 (+68 ms)Preparing flush snapshotting stores in d9a2369468b78e2bdc86c3499284cfae at 1732423748182Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732423748183 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. at 1732423748183Flushing d9a2369468b78e2bdc86c3499284cfae/a: creating writer at 1732423748183Flushing d9a2369468b78e2bdc86c3499284cfae/a: appending metadata at 1732423748206 (+23 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/a: closing flushed file at 1732423748206Flushing d9a2369468b78e2bdc86c3499284cfae/b: creating writer at 1732423748231 (+25 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/b: appending metadata at 1732423748266 (+35 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/b: closing flushed file at 1732423748266Flushing d9a2369468b78e2bdc86c3499284cfae/c: creating writer at 1732423748281 (+15 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/c: appending metadata at 1732423748297 (+16 ms)Flushing d9a2369468b78e2bdc86c3499284cfae/c: closing flushed file at 1732423748298 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c002aac: reopening flushed file at 1732423748322 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48f1c22c: reopening flushed file at 1732423748333 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23f831d0: reopening flushed file at 1732423748340 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d9a2369468b78e2bdc86c3499284cfae in 167ms, sequenceid=66, compaction requested=false; wal=null at 1732423748349 (+9 ms)Cleaning up temporary data from old regions at 1732423748351 (+2 ms)Region opened successfully at 1732423748357 (+6 ms) 2024-11-24T04:49:08,375 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9a2369468b78e2bdc86c3499284cfae, disabling compactions & flushes 2024-11-24T04:49:08,375 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:08,375 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:08,375 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. after waiting 0 ms 2024-11-24T04:49:08,375 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:08,382 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423743298.d9a2369468b78e2bdc86c3499284cfae. 2024-11-24T04:49:08,382 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9a2369468b78e2bdc86c3499284cfae: Waiting for close lock at 1732423748375Disabling compacts and flushes for region at 1732423748375Disabling writes for close at 1732423748375Writing region close event to WAL at 1732423748382 (+7 ms)Closed at 1732423748382 2024-11-24T04:49:08,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741934_1114 (size=175) 2024-11-24T04:49:08,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741934_1114 (size=175) 2024-11-24T04:49:08,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741934_1114 (size=175) 2024-11-24T04:49:08,388 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:49:08,388 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423748052) 2024-11-24T04:49:08,405 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=442 (was 439) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-880652537_22 at /127.0.0.1:52018 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-880652537_22 at /127.0.0.1:43208 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:43985 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-880652537_22 at /127.0.0.1:59278 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43985 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=531 (was 534), ProcessCount=11 (was 11), AvailableMemoryMB=10184 (was 10197) 2024-11-24T04:49:08,405 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-24T04:49:08,405 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T04:49:08,405 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:49:08,406 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:49:08,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:08,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:08,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:49:08,407 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T04:49:08,408 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=826943530, stopped=false 2024-11-24T04:49:08,408 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4464c5b832df,38057,1732423703360 2024-11-24T04:49:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:49:08,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:49:08,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:49:08,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:49:08,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:49:08,474 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:49:08,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:49:08,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:49:08,475 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:49:08,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:49:08,476 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:49:08,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:49:08,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:08,477 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4464c5b832df,42899,1732423704094' ***** 2024-11-24T04:49:08,477 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:49:08,478 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4464c5b832df,33335,1732423704223' ***** 2024-11-24T04:49:08,478 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:49:08,478 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(3091): Received CLOSE for 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(959): stopping server 4464c5b832df,42899,1732423704094 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:49:08,478 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3002ca1159d88d4ded5fb6e5f70fe27e, disabling compactions & flushes 2024-11-24T04:49:08,478 INFO [RS:0;4464c5b832df:42899 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4464c5b832df:42899. 2024-11-24T04:49:08,478 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:49:08,478 DEBUG [RS:0;4464c5b832df:42899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:49:08,478 DEBUG [RS:0;4464c5b832df:42899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:08,478 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:49:08,479 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. after waiting 0 ms 2024-11-24T04:49:08,479 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:49:08,479 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:49:08,479 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1325): Online Regions={3002ca1159d88d4ded5fb6e5f70fe27e=testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e.} 2024-11-24T04:49:08,479 DEBUG [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1351): Waiting on 3002ca1159d88d4ded5fb6e5f70fe27e 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:49:08,479 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(959): stopping server 4464c5b832df,33335,1732423704223 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4464c5b832df:33335. 2024-11-24T04:49:08,479 DEBUG [RS:2;4464c5b832df:33335 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:49:08,479 DEBUG [RS:2;4464c5b832df:33335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:08,479 INFO [RS:2;4464c5b832df:33335 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:49:08,484 INFO [RS:2;4464c5b832df:33335 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:49:08,484 INFO [RS:2;4464c5b832df:33335 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:49:08,485 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T04:49:08,501 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:49:08,501 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T04:49:08,501 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T04:49:08,502 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:49:08,502 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:49:08,502 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:49:08,502 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:49:08,502 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:49:08,502 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-24T04:49:08,527 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3002ca1159d88d4ded5fb6e5f70fe27e/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T04:49:08,528 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:49:08,528 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3002ca1159d88d4ded5fb6e5f70fe27e: Waiting for close lock at 1732423748478Running coprocessor pre-close hooks at 1732423748478Disabling compacts and flushes for region at 1732423748478Disabling writes for close at 1732423748479 (+1 ms)Writing region close event to WAL at 1732423748505 (+26 ms)Running coprocessor post-close hooks at 1732423748528 (+23 ms)Closed at 1732423748528 2024-11-24T04:49:08,528 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e. 2024-11-24T04:49:08,532 INFO [regionserver/4464c5b832df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:49:08,535 INFO [regionserver/4464c5b832df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:49:08,556 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/info/522f8afe7ed94e698fe6d70507b2eeba is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732423722021.3002ca1159d88d4ded5fb6e5f70fe27e./info:regioninfo/1732423725852/Put/seqid=0 2024-11-24T04:49:08,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:49:08,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:49:08,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:49:08,679 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(976): stopping server 4464c5b832df,42899,1732423704094; all regions closed. 2024-11-24T04:49:08,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741835_1011 (size=3621) 2024-11-24T04:49:08,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741835_1011 (size=3621) 2024-11-24T04:49:08,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741835_1011 (size=3621) 2024-11-24T04:49:08,689 DEBUG [RS:0;4464c5b832df:42899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs 2024-11-24T04:49:08,689 INFO [RS:0;4464c5b832df:42899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4464c5b832df%2C42899%2C1732423704094:(num 1732423706054) 2024-11-24T04:49:08,689 DEBUG [RS:0;4464c5b832df:42899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:08,689 INFO [RS:0;4464c5b832df:42899 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:49:08,689 INFO [RS:0;4464c5b832df:42899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:49:08,690 INFO [RS:0;4464c5b832df:42899 {}] hbase.ChoreService(370): Chore service for: regionserver/4464c5b832df:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T04:49:08,690 INFO [RS:0;4464c5b832df:42899 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:49:08,690 INFO [RS:0;4464c5b832df:42899 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:49:08,690 INFO [regionserver/4464c5b832df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:49:08,690 INFO [RS:0;4464c5b832df:42899 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:49:08,690 INFO [RS:0;4464c5b832df:42899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:49:08,690 INFO [RS:0;4464c5b832df:42899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42899 2024-11-24T04:49:08,701 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T04:49:08,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:49:08,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4464c5b832df,42899,1732423704094 2024-11-24T04:49:08,703 INFO [RS:0;4464c5b832df:42899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:49:08,704 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4464c5b832df,42899,1732423704094] 2024-11-24T04:49:08,724 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4464c5b832df,42899,1732423704094 already deleted, retry=false 2024-11-24T04:49:08,724 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4464c5b832df,42899,1732423704094 expired; onlineServers=1 2024-11-24T04:49:08,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:49:08,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42899-0x1016b2bd3dc0001, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:49:08,814 INFO [RS:0;4464c5b832df:42899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:49:08,814 INFO [RS:0;4464c5b832df:42899 {}] regionserver.HRegionServer(1031): Exiting; stopping=4464c5b832df,42899,1732423704094; zookeeper connection closed. 2024-11-24T04:49:08,814 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e2a13d0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e2a13d0 2024-11-24T04:49:08,901 DEBUG [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T04:49:08,977 INFO [regionserver/4464c5b832df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T04:49:08,977 INFO [regionserver/4464c5b832df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T04:49:08,992 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/info/522f8afe7ed94e698fe6d70507b2eeba 2024-11-24T04:49:09,013 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/ns/8c639a5d525e4c4b99a9a7fbceab669a is 43, key is default/ns:d/1732423706683/Put/seqid=0 2024-11-24T04:49:09,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:49:09,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:49:09,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:49:09,025 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/ns/8c639a5d525e4c4b99a9a7fbceab669a 2024-11-24T04:49:09,044 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/table/ca655261085443b392b3782e44f9d449 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732423722482/Put/seqid=0 2024-11-24T04:49:09,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:49:09,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:49:09,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:49:09,060 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/table/ca655261085443b392b3782e44f9d449 2024-11-24T04:49:09,066 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/info/522f8afe7ed94e698fe6d70507b2eeba as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/info/522f8afe7ed94e698fe6d70507b2eeba 2024-11-24T04:49:09,073 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/info/522f8afe7ed94e698fe6d70507b2eeba, entries=18, sequenceid=21, filesize=8.0 K 2024-11-24T04:49:09,074 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/ns/8c639a5d525e4c4b99a9a7fbceab669a as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/ns/8c639a5d525e4c4b99a9a7fbceab669a 2024-11-24T04:49:09,080 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/ns/8c639a5d525e4c4b99a9a7fbceab669a, entries=2, sequenceid=21, filesize=5.0 K 2024-11-24T04:49:09,081 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/.tmp/table/ca655261085443b392b3782e44f9d449 as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/table/ca655261085443b392b3782e44f9d449 2024-11-24T04:49:09,086 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/table/ca655261085443b392b3782e44f9d449, entries=2, sequenceid=21, filesize=5.3 K 2024-11-24T04:49:09,087 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 585ms, sequenceid=21, compaction requested=false 2024-11-24T04:49:09,093 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-24T04:49:09,094 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:49:09,094 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:49:09,094 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732423748502Running coprocessor pre-close hooks at 1732423748502Disabling compacts and flushes for region at 1732423748502Disabling writes for close at 1732423748502Obtaining lock to block concurrent updates at 1732423748502Preparing flush snapshotting stores in 1588230740 at 1732423748502Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732423748503 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732423748504 (+1 ms)Flushing 1588230740/info: creating writer at 1732423748504Flushing 1588230740/info: appending metadata at 1732423748556 (+52 ms)Flushing 1588230740/info: closing flushed file at 1732423748556Flushing 1588230740/ns: creating writer at 1732423748997 (+441 ms)Flushing 1588230740/ns: appending metadata at 1732423749012 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732423749012Flushing 1588230740/table: creating writer at 1732423749030 (+18 ms)Flushing 1588230740/table: appending metadata at 1732423749044 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732423749044Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@235e1f51: reopening flushed file at 1732423749066 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d968e9: reopening flushed file at 1732423749073 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bff6b93: reopening flushed file at 1732423749080 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 585ms, sequenceid=21, compaction requested=false at 1732423749087 (+7 ms)Writing region close event to WAL at 1732423749088 (+1 ms)Running coprocessor post-close hooks at 1732423749093 (+5 ms)Closed at 1732423749094 (+1 ms) 2024-11-24T04:49:09,094 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T04:49:09,102 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(976): stopping server 4464c5b832df,33335,1732423704223; all regions closed. 2024-11-24T04:49:09,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741836_1012 (size=9487) 2024-11-24T04:49:09,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741836_1012 (size=9487) 2024-11-24T04:49:09,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741836_1012 (size=9487) 2024-11-24T04:49:09,108 DEBUG [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs 2024-11-24T04:49:09,109 INFO [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4464c5b832df%2C33335%2C1732423704223.meta:.meta(num 1732423706463) 2024-11-24T04:49:09,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741833_1009 (size=175) 2024-11-24T04:49:09,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741833_1009 (size=175) 2024-11-24T04:49:09,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741833_1009 (size=175) 2024-11-24T04:49:09,115 DEBUG [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/oldWALs 2024-11-24T04:49:09,116 INFO [RS:2;4464c5b832df:33335 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4464c5b832df%2C33335%2C1732423704223:(num 1732423706054) 2024-11-24T04:49:09,116 DEBUG [RS:2;4464c5b832df:33335 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:49:09,116 INFO [RS:2;4464c5b832df:33335 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:49:09,116 INFO [RS:2;4464c5b832df:33335 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:49:09,116 INFO [RS:2;4464c5b832df:33335 {}] hbase.ChoreService(370): Chore service for: regionserver/4464c5b832df:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:49:09,116 INFO [RS:2;4464c5b832df:33335 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:49:09,116 INFO [RS:2;4464c5b832df:33335 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33335 2024-11-24T04:49:09,117 INFO [regionserver/4464c5b832df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:49:09,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:49:09,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4464c5b832df,33335,1732423704223 2024-11-24T04:49:09,125 INFO [RS:2;4464c5b832df:33335 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:49:09,125 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4464c5b832df,33335,1732423704223] 2024-11-24T04:49:09,145 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4464c5b832df,33335,1732423704223 already deleted, retry=false 2024-11-24T04:49:09,145 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4464c5b832df,33335,1732423704223 expired; onlineServers=0 2024-11-24T04:49:09,146 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4464c5b832df,38057,1732423703360' ***** 2024-11-24T04:49:09,146 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T04:49:09,146 INFO [M:0;4464c5b832df:38057 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:49:09,146 INFO [M:0;4464c5b832df:38057 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:49:09,146 DEBUG [M:0;4464c5b832df:38057 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T04:49:09,146 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T04:49:09,146 DEBUG [M:0;4464c5b832df:38057 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T04:49:09,146 DEBUG [master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.large.0-1732423705673 {}] cleaner.HFileCleaner(306): Exit Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.large.0-1732423705673,5,FailOnTimeoutGroup] 2024-11-24T04:49:09,146 INFO [M:0;4464c5b832df:38057 {}] hbase.ChoreService(370): Chore service for: master/4464c5b832df:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T04:49:09,146 DEBUG [master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.small.0-1732423705673 {}] cleaner.HFileCleaner(306): Exit Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.small.0-1732423705673,5,FailOnTimeoutGroup] 2024-11-24T04:49:09,146 INFO [M:0;4464c5b832df:38057 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:49:09,146 DEBUG [M:0;4464c5b832df:38057 {}] master.HMaster(1795): Stopping service threads 2024-11-24T04:49:09,147 INFO [M:0;4464c5b832df:38057 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T04:49:09,147 INFO [M:0;4464c5b832df:38057 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:49:09,147 INFO [M:0;4464c5b832df:38057 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T04:49:09,147 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T04:49:09,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T04:49:09,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:49:09,156 DEBUG [M:0;4464c5b832df:38057 {}] zookeeper.ZKUtil(347): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T04:49:09,156 WARN [M:0;4464c5b832df:38057 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T04:49:09,157 INFO [M:0;4464c5b832df:38057 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/.lastflushedseqids 2024-11-24T04:49:09,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:49:09,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:49:09,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:49:09,183 INFO [M:0;4464c5b832df:38057 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T04:49:09,183 INFO [M:0;4464c5b832df:38057 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T04:49:09,183 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:49:09,183 INFO [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:49:09,183 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:49:09,183 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:49:09,183 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:49:09,183 INFO [M:0;4464c5b832df:38057 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-24T04:49:09,205 DEBUG [M:0;4464c5b832df:38057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2a0781bb32c24c99adf023eb45f57c04 is 82, key is hbase:meta,,1/info:regioninfo/1732423706567/Put/seqid=0 2024-11-24T04:49:09,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:49:09,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:49:09,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:49:09,212 INFO [M:0;4464c5b832df:38057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2a0781bb32c24c99adf023eb45f57c04 2024-11-24T04:49:09,232 DEBUG [M:0;4464c5b832df:38057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36647280b13b4c4ead7dcf51370a6cab is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732423722493/Put/seqid=0 2024-11-24T04:49:09,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:49:09,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33335-0x1016b2bd3dc0003, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:49:09,235 INFO [RS:2;4464c5b832df:33335 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:49:09,235 INFO [RS:2;4464c5b832df:33335 {}] regionserver.HRegionServer(1031): Exiting; stopping=4464c5b832df,33335,1732423704223; zookeeper connection closed. 2024-11-24T04:49:09,235 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a01dc94 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a01dc94 2024-11-24T04:49:09,236 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T04:49:09,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741943_1123 (size=7755) 2024-11-24T04:49:09,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741943_1123 (size=7755) 2024-11-24T04:49:09,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741943_1123 (size=7755) 2024-11-24T04:49:09,252 INFO [M:0;4464c5b832df:38057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36647280b13b4c4ead7dcf51370a6cab 2024-11-24T04:49:09,257 INFO [M:0;4464c5b832df:38057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 36647280b13b4c4ead7dcf51370a6cab 2024-11-24T04:49:09,277 DEBUG [M:0;4464c5b832df:38057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0aab7387da354f8484120091158a3e7d is 69, key is 4464c5b832df,33335,1732423704223/rs:state/1732423705756/Put/seqid=0 2024-11-24T04:49:09,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741944_1124 (size=5440) 2024-11-24T04:49:09,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741944_1124 (size=5440) 2024-11-24T04:49:09,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741944_1124 (size=5440) 2024-11-24T04:49:09,294 INFO [M:0;4464c5b832df:38057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0aab7387da354f8484120091158a3e7d 2024-11-24T04:49:09,299 INFO [M:0;4464c5b832df:38057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0aab7387da354f8484120091158a3e7d 2024-11-24T04:49:09,300 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2a0781bb32c24c99adf023eb45f57c04 as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2a0781bb32c24c99adf023eb45f57c04 2024-11-24T04:49:09,305 INFO [M:0;4464c5b832df:38057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2a0781bb32c24c99adf023eb45f57c04, entries=8, sequenceid=168, filesize=5.5 K 2024-11-24T04:49:09,306 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/36647280b13b4c4ead7dcf51370a6cab as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/36647280b13b4c4ead7dcf51370a6cab 2024-11-24T04:49:09,311 INFO [M:0;4464c5b832df:38057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 36647280b13b4c4ead7dcf51370a6cab 2024-11-24T04:49:09,311 INFO [M:0;4464c5b832df:38057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/36647280b13b4c4ead7dcf51370a6cab, entries=17, sequenceid=168, filesize=7.6 K 2024-11-24T04:49:09,312 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0aab7387da354f8484120091158a3e7d as hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0aab7387da354f8484120091158a3e7d 2024-11-24T04:49:09,317 INFO [M:0;4464c5b832df:38057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0aab7387da354f8484120091158a3e7d 2024-11-24T04:49:09,317 INFO [M:0;4464c5b832df:38057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43985/user/jenkins/test-data/73988220-5500-48df-c103-79a374d24269/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0aab7387da354f8484120091158a3e7d, entries=3, sequenceid=168, filesize=5.3 K 2024-11-24T04:49:09,318 INFO [M:0;4464c5b832df:38057 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=168, compaction requested=false 2024-11-24T04:49:09,319 INFO [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:49:09,319 DEBUG [M:0;4464c5b832df:38057 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732423749183Disabling compacts and flushes for region at 1732423749183Disabling writes for close at 1732423749183Obtaining lock to block concurrent updates at 1732423749183Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732423749183Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732423749184 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732423749185 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732423749185Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732423749204 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732423749204Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732423749216 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732423749231 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732423749231Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732423749257 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732423749276 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732423749276Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f0bceca: reopening flushed file at 1732423749300 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d1be21: reopening flushed file at 1732423749305 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ad5df26: reopening flushed file at 1732423749311 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=168, compaction requested=false at 1732423749318 (+7 ms)Writing region close event to WAL at 1732423749319 (+1 ms)Closed at 1732423749319 2024-11-24T04:49:09,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45441 is added to blk_1073741830_1006 (size=83606) 2024-11-24T04:49:09,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35959 is added to blk_1073741830_1006 (size=83606) 2024-11-24T04:49:09,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43379 is added to blk_1073741830_1006 (size=83606) 2024-11-24T04:49:09,332 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:49:09,333 INFO [M:0;4464c5b832df:38057 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T04:49:09,333 INFO [M:0;4464c5b832df:38057 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38057 2024-11-24T04:49:09,333 INFO [M:0;4464c5b832df:38057 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:49:09,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:49:09,445 INFO [M:0;4464c5b832df:38057 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:49:09,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38057-0x1016b2bd3dc0000, quorum=127.0.0.1:57729, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:49:09,452 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726761 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726761 (inode 16655) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423726110/wal.1732423726761 (inode 16655) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:49:09,453 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423717326/wal.1732423717482 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:09,453 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423743297/wal.1732423743694 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:09,456 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423736612 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423736612 (inode 16678) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423726993/wal.1732423736612 (inode 16678) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:49:09,457 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423708231/wal.1732423708442 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:09,460 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423737462 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423737462 (inode 16704) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423736840/wal.1732423737462 (inode 16704) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:49:09,460 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423737666/wal.1732423737848 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:49:09,462 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal.1732423708008 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal.1732423708008 (inode 16485) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423707640/wal.1732423708008 (inode 16485) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:49:09,465 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal.1732423707382 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal.1732423707382 (inode 16462) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423706952/wal.1732423707382 (inode 16462) Holder DFSClient_NONMAPREDUCE_-680574894_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:49:09,478 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@773b4a40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:49:09,480 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:49:09,480 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:49:09,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:49:09,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,STOPPED} 2024-11-24T04:49:09,484 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:49:09,484 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:49:09,484 WARN [BP-2110872418-172.17.0.2-1732423698697 heartbeating to localhost/127.0.0.1:43985 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:49:09,484 WARN [BP-2110872418-172.17.0.2-1732423698697 heartbeating to localhost/127.0.0.1:43985 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2110872418-172.17.0.2-1732423698697 (Datanode Uuid 89350da1-96f6-4e8b-bcd1-0f12b66d8f37) service to localhost/127.0.0.1:43985 2024-11-24T04:49:09,486 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data5/current/BP-2110872418-172.17.0.2-1732423698697 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:49:09,486 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data6/current/BP-2110872418-172.17.0.2-1732423698697 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:49:09,486 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:49:09,493 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75d96791{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:49:09,494 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:49:09,494 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:49:09,494 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:49:09,494 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,STOPPED} 2024-11-24T04:49:09,495 WARN [BP-2110872418-172.17.0.2-1732423698697 heartbeating to localhost/127.0.0.1:43985 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:49:09,495 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:49:09,495 WARN [BP-2110872418-172.17.0.2-1732423698697 heartbeating to localhost/127.0.0.1:43985 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2110872418-172.17.0.2-1732423698697 (Datanode Uuid 5230f75c-0cac-461f-9046-29cca0650db0) service to localhost/127.0.0.1:43985 2024-11-24T04:49:09,495 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:49:09,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data3/current/BP-2110872418-172.17.0.2-1732423698697 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:49:09,496 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data4/current/BP-2110872418-172.17.0.2-1732423698697 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:49:09,496 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:49:09,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d480c14{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:49:09,502 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:49:09,502 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:49:09,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:49:09,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,STOPPED} 2024-11-24T04:49:09,504 WARN [BP-2110872418-172.17.0.2-1732423698697 heartbeating to localhost/127.0.0.1:43985 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:49:09,504 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:49:09,504 WARN [BP-2110872418-172.17.0.2-1732423698697 heartbeating to localhost/127.0.0.1:43985 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2110872418-172.17.0.2-1732423698697 (Datanode Uuid 152bafc3-db70-4c2a-8428-6f5a414ad0e5) service to localhost/127.0.0.1:43985 2024-11-24T04:49:09,504 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:49:09,504 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data1/current/BP-2110872418-172.17.0.2-1732423698697 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:49:09,505 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/cluster_4fc7af09-5d01-0c36-8da4-3f6c65420672/data/data2/current/BP-2110872418-172.17.0.2-1732423698697 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:49:09,508 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:49:09,514 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14f32243{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:49:09,515 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:49:09,515 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:49:09,515 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:49:09,515 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2b764948-4968-4e4f-f4d9-ef6941f660e9/hadoop.log.dir/,STOPPED} 2024-11-24T04:49:09,525 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T04:49:09,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down