2024-11-27 01:25:34,419 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-27 01:25:34,430 main DEBUG Took 0.009461 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 01:25:34,430 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 01:25:34,430 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 01:25:34,432 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 01:25:34,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,441 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 01:25:34,458 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,460 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,461 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,463 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,464 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,465 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,467 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,468 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,469 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,470 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,472 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 01:25:34,473 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,474 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 01:25:34,475 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 01:25:34,477 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 01:25:34,479 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 01:25:34,480 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 01:25:34,481 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 01:25:34,481 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 01:25:34,495 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 01:25:34,498 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 01:25:34,500 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 01:25:34,501 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 01:25:34,501 main DEBUG createAppenders(={Console}) 2024-11-27 01:25:34,502 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-27 01:25:34,502 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-27 01:25:34,503 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-27 01:25:34,503 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 01:25:34,504 main DEBUG OutputStream closed 2024-11-27 01:25:34,504 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 01:25:34,505 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 01:25:34,505 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-27 01:25:34,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 01:25:34,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 01:25:34,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 01:25:34,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 01:25:34,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 01:25:34,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 01:25:34,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 01:25:34,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 01:25:34,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 01:25:34,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 01:25:34,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 01:25:34,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 01:25:34,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 01:25:34,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 01:25:34,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 01:25:34,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 01:25:34,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 01:25:34,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 01:25:34,627 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 01:25:34,627 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-27 01:25:34,628 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 01:25:34,629 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-27T01:25:34,947 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f 2024-11-27 01:25:34,951 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 01:25:34,951 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T01:25:34,961 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-27T01:25:34,970 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-27T01:25:35,010 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T01:25:35,094 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T01:25:35,095 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T01:25:35,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T01:25:35,140 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9, deleteOnExit=true 2024-11-27T01:25:35,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T01:25:35,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/test.cache.data in system properties and HBase conf 2024-11-27T01:25:35,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T01:25:35,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir in system properties and HBase conf 2024-11-27T01:25:35,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T01:25:35,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T01:25:35,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T01:25:35,243 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T01:25:35,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T01:25:35,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:25:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T01:25:35,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T01:25:35,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:25:35,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T01:25:35,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T01:25:35,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T01:25:35,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:25:35,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T01:25:35,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/nfs.dump.dir in system properties and HBase conf 2024-11-27T01:25:35,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/java.io.tmpdir in system properties and HBase conf 2024-11-27T01:25:35,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T01:25:35,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T01:25:35,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T01:25:36,284 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T01:25:36,379 INFO [Time-limited test {}] log.Log(170): Logging initialized @2816ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T01:25:36,454 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:25:36,516 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:25:36,537 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:25:36,537 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:25:36,538 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T01:25:36,557 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:25:36,560 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:25:36,561 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:25:36,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14f32243{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/java.io.tmpdir/jetty-localhost-36011-hadoop-hdfs-3_4_1-tests_jar-_-any-16347521063865369959/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:25:36,754 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:36011} 2024-11-27T01:25:36,754 INFO [Time-limited test {}] server.Server(415): Started @3191ms 2024-11-27T01:25:37,133 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:25:37,139 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:25:37,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:25:37,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:25:37,141 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T01:25:37,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:25:37,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:25:37,268 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d480c14{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/java.io.tmpdir/jetty-localhost-32981-hadoop-hdfs-3_4_1-tests_jar-_-any-13506639900908989024/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:25:37,269 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:32981} 2024-11-27T01:25:37,269 INFO [Time-limited test {}] server.Server(415): Started @3706ms 2024-11-27T01:25:37,331 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:25:37,472 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:25:37,477 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:25:37,484 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:25:37,485 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:25:37,485 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T01:25:37,486 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:25:37,486 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:25:37,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75d96791{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/java.io.tmpdir/jetty-localhost-41967-hadoop-hdfs-3_4_1-tests_jar-_-any-12419167504857886856/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:25:37,605 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:41967} 2024-11-27T01:25:37,606 INFO [Time-limited test {}] server.Server(415): Started @4042ms 2024-11-27T01:25:37,608 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:25:37,646 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T01:25:37,651 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T01:25:37,652 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T01:25:37,652 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T01:25:37,652 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T01:25:37,653 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,AVAILABLE} 2024-11-27T01:25:37,654 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T01:25:37,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@773b4a40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/java.io.tmpdir/jetty-localhost-46311-hadoop-hdfs-3_4_1-tests_jar-_-any-18034942593283174571/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:25:37,782 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:46311} 2024-11-27T01:25:37,782 INFO [Time-limited test {}] server.Server(415): Started @4219ms 2024-11-27T01:25:37,785 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T01:25:37,795 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data3/current/BP-879573077-172.17.0.2-1732670735999/current, will proceed with Du for space computation calculation, 2024-11-27T01:25:37,795 WARN [Thread-111 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data2/current/BP-879573077-172.17.0.2-1732670735999/current, will proceed with Du for space computation calculation, 2024-11-27T01:25:37,795 WARN [Thread-110 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data1/current/BP-879573077-172.17.0.2-1732670735999/current, will proceed with Du for space computation calculation, 2024-11-27T01:25:37,795 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data4/current/BP-879573077-172.17.0.2-1732670735999/current, will proceed with Du for space computation calculation, 2024-11-27T01:25:37,841 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:25:37,841 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:25:37,918 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x463af61701c47147 with lease ID 0x8b12599cd9a71b97: Processing first storage report for DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9 from datanode DatanodeRegistration(127.0.0.1:39261, datanodeUuid=971d4915-7e58-4935-b45e-daabb70295fa, infoPort=41833, infoSecurePort=0, ipcPort=45933, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999) 2024-11-27T01:25:37,919 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x463af61701c47147 with lease ID 0x8b12599cd9a71b97: from storage DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9 node DatanodeRegistration(127.0.0.1:39261, datanodeUuid=971d4915-7e58-4935-b45e-daabb70295fa, infoPort=41833, infoSecurePort=0, ipcPort=45933, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T01:25:37,919 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1a8cf75f4b92b976 with lease ID 0x8b12599cd9a71b96: Processing first storage report for DS-514f1085-51f8-4d8d-97bc-002604feb161 from datanode DatanodeRegistration(127.0.0.1:44005, datanodeUuid=1c2e6c23-04a2-455c-8937-48d461463669, infoPort=35465, infoSecurePort=0, ipcPort=45477, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999) 2024-11-27T01:25:37,919 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a8cf75f4b92b976 with lease ID 0x8b12599cd9a71b96: from storage DS-514f1085-51f8-4d8d-97bc-002604feb161 node DatanodeRegistration(127.0.0.1:44005, datanodeUuid=1c2e6c23-04a2-455c-8937-48d461463669, infoPort=35465, infoSecurePort=0, ipcPort=45477, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:25:37,920 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x463af61701c47147 with lease ID 0x8b12599cd9a71b97: Processing first storage report for DS-90f2eed1-f9f9-4909-a89c-2d81a0a03ab6 from datanode DatanodeRegistration(127.0.0.1:39261, datanodeUuid=971d4915-7e58-4935-b45e-daabb70295fa, infoPort=41833, infoSecurePort=0, ipcPort=45933, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999) 2024-11-27T01:25:37,920 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x463af61701c47147 with lease ID 0x8b12599cd9a71b97: from storage DS-90f2eed1-f9f9-4909-a89c-2d81a0a03ab6 node DatanodeRegistration(127.0.0.1:39261, datanodeUuid=971d4915-7e58-4935-b45e-daabb70295fa, infoPort=41833, infoSecurePort=0, ipcPort=45933, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T01:25:37,920 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1a8cf75f4b92b976 with lease ID 0x8b12599cd9a71b96: Processing first storage report for DS-72815b32-26ac-4c9b-babc-4ab637257036 from datanode DatanodeRegistration(127.0.0.1:44005, datanodeUuid=1c2e6c23-04a2-455c-8937-48d461463669, infoPort=35465, infoSecurePort=0, ipcPort=45477, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999) 2024-11-27T01:25:37,920 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1a8cf75f4b92b976 with lease ID 0x8b12599cd9a71b96: from storage DS-72815b32-26ac-4c9b-babc-4ab637257036 node DatanodeRegistration(127.0.0.1:44005, datanodeUuid=1c2e6c23-04a2-455c-8937-48d461463669, infoPort=35465, infoSecurePort=0, ipcPort=45477, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:25:37,920 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data5/current/BP-879573077-172.17.0.2-1732670735999/current, will proceed with Du for space computation calculation, 2024-11-27T01:25:37,921 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data6/current/BP-879573077-172.17.0.2-1732670735999/current, will proceed with Du for space computation calculation, 2024-11-27T01:25:37,950 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T01:25:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12d2868c1e636635 with lease ID 0x8b12599cd9a71b98: Processing first storage report for DS-61b35176-9b92-4ab0-bf15-999f3f0320fa from datanode DatanodeRegistration(127.0.0.1:45665, datanodeUuid=246cac20-e37a-4e3e-9564-168acfc5ca14, infoPort=38391, infoSecurePort=0, ipcPort=34577, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999) 2024-11-27T01:25:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12d2868c1e636635 with lease ID 0x8b12599cd9a71b98: from storage DS-61b35176-9b92-4ab0-bf15-999f3f0320fa node DatanodeRegistration(127.0.0.1:45665, datanodeUuid=246cac20-e37a-4e3e-9564-168acfc5ca14, infoPort=38391, infoSecurePort=0, ipcPort=34577, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T01:25:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x12d2868c1e636635 with lease ID 0x8b12599cd9a71b98: Processing first storage report for DS-9e781d97-2308-4432-9f88-e0cdd7817e1e from datanode DatanodeRegistration(127.0.0.1:45665, datanodeUuid=246cac20-e37a-4e3e-9564-168acfc5ca14, infoPort=38391, infoSecurePort=0, ipcPort=34577, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999) 2024-11-27T01:25:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x12d2868c1e636635 with lease ID 0x8b12599cd9a71b98: from storage DS-9e781d97-2308-4432-9f88-e0cdd7817e1e node DatanodeRegistration(127.0.0.1:45665, datanodeUuid=246cac20-e37a-4e3e-9564-168acfc5ca14, infoPort=38391, infoSecurePort=0, ipcPort=34577, storageInfo=lv=-57;cid=testClusterID;nsid=554125827;c=1732670735999), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T01:25:38,143 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f 2024-11-27T01:25:38,225 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/zookeeper_0, clientPort=58346, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T01:25:38,234 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58346 2024-11-27T01:25:38,250 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:38,254 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:25:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:25:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741825_1001 (size=7) 2024-11-27T01:25:38,880 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1 with version=8 2024-11-27T01:25:38,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/hbase-staging 2024-11-27T01:25:39,178 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:25:39,188 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,189 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,193 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:25:39,194 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,194 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:25:39,330 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T01:25:39,389 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T01:25:39,401 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T01:25:39,406 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:25:39,442 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14397 (auto-detected) 2024-11-27T01:25:39,444 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T01:25:39,463 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37429 2024-11-27T01:25:39,484 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37429 connecting to ZooKeeper ensemble=127.0.0.1:58346 2024-11-27T01:25:39,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:374290x0, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:25:39,518 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37429-0x101570739b00000 connected 2024-11-27T01:25:39,544 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,548 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:25:39,562 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1, hbase.cluster.distributed=false 2024-11-27T01:25:39,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:25:39,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37429 2024-11-27T01:25:39,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37429 2024-11-27T01:25:39,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37429 2024-11-27T01:25:39,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37429 2024-11-27T01:25:39,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37429 2024-11-27T01:25:39,715 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:25:39,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,717 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:25:39,717 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,718 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:25:39,720 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:25:39,722 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:25:39,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41943 2024-11-27T01:25:39,725 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41943 connecting to ZooKeeper ensemble=127.0.0.1:58346 2024-11-27T01:25:39,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419430x0, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:25:39,737 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:25:39,737 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41943-0x101570739b00001 connected 2024-11-27T01:25:39,741 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:25:39,749 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:25:39,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:25:39,756 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:25:39,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41943 2024-11-27T01:25:39,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41943 2024-11-27T01:25:39,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41943 2024-11-27T01:25:39,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41943 2024-11-27T01:25:39,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41943 2024-11-27T01:25:39,776 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:25:39,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,777 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:25:39,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:25:39,777 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:25:39,778 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:25:39,778 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39245 2024-11-27T01:25:39,780 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39245 connecting to ZooKeeper ensemble=127.0.0.1:58346 2024-11-27T01:25:39,781 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,784 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392450x0, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:25:39,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39245-0x101570739b00002 connected 2024-11-27T01:25:39,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:25:39,790 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:25:39,792 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:25:39,793 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:25:39,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:25:39,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39245 2024-11-27T01:25:39,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39245 2024-11-27T01:25:39,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39245 2024-11-27T01:25:39,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39245 2024-11-27T01:25:39,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39245 2024-11-27T01:25:39,814 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d6f26a4c136:0 server-side Connection retries=45 2024-11-27T01:25:39,815 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,815 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,815 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T01:25:39,815 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T01:25:39,816 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T01:25:39,816 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T01:25:39,816 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T01:25:39,817 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43859 2024-11-27T01:25:39,819 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43859 connecting to ZooKeeper ensemble=127.0.0.1:58346 2024-11-27T01:25:39,821 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,824 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:39,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438590x0, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T01:25:39,832 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:438590x0, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:25:39,832 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43859-0x101570739b00003 connected 2024-11-27T01:25:39,832 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T01:25:39,833 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T01:25:39,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T01:25:39,837 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T01:25:39,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43859 2024-11-27T01:25:39,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43859 2024-11-27T01:25:39,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43859 2024-11-27T01:25:39,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43859 2024-11-27T01:25:39,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43859 2024-11-27T01:25:39,867 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7d6f26a4c136:37429 2024-11-27T01:25:39,868 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:39,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,879 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T01:25:39,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:39,901 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T01:25:39,902 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7d6f26a4c136,37429,1732670738986 from backup master directory 2024-11-27T01:25:39,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:39,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T01:25:39,906 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:25:39,906 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:39,909 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T01:25:39,911 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T01:25:39,967 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/hbase.id] with ID: 09523c17-5740-4230-8a95-38d6b0bf24fa 2024-11-27T01:25:39,967 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/.tmp/hbase.id 2024-11-27T01:25:39,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:25:39,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:25:39,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741826_1002 (size=42) 2024-11-27T01:25:39,982 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/.tmp/hbase.id]:[hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/hbase.id] 2024-11-27T01:25:40,025 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T01:25:40,029 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T01:25:40,048 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-27T01:25:40,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:25:40,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:25:40,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741827_1003 (size=196) 2024-11-27T01:25:40,088 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:25:40,090 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T01:25:40,096 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:25:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:25:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741828_1004 (size=1189) 2024-11-27T01:25:40,147 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store 2024-11-27T01:25:40,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:25:40,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:25:40,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741829_1005 (size=34) 2024-11-27T01:25:40,177 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T01:25:40,181 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:40,183 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:25:40,183 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:25:40,183 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:25:40,185 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:25:40,185 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:25:40,186 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:25:40,187 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732670740183Disabling compacts and flushes for region at 1732670740183Disabling writes for close at 1732670740185 (+2 ms)Writing region close event to WAL at 1732670740186 (+1 ms)Closed at 1732670740186 2024-11-27T01:25:40,189 WARN [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/.initializing 2024-11-27T01:25:40,189 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/WALs/7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:40,197 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:40,211 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C37429%2C1732670738986, suffix=, logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/WALs/7d6f26a4c136,37429,1732670738986, archiveDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/oldWALs, maxLogs=10 2024-11-27T01:25:40,242 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/WALs/7d6f26a4c136,37429,1732670738986/7d6f26a4c136%2C37429%2C1732670738986.1732670740216, exclude list is [], retry=0 2024-11-27T01:25:40,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:40,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:40,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:40,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:40,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T01:25:40,307 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-27T01:25:40,369 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/WALs/7d6f26a4c136,37429,1732670738986/7d6f26a4c136%2C37429%2C1732670738986.1732670740216 2024-11-27T01:25:40,370 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:40,371 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:40,371 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:40,374 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,375 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T01:25:40,442 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:40,445 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T01:25:40,448 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:40,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T01:25:40,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:40,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T01:25:40,456 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:40,457 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,460 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,462 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,466 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,467 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,470 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:25:40,473 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T01:25:40,478 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:40,479 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74211640, jitterRate=0.10583961009979248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:25:40,485 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732670740387Initializing all the Stores at 1732670740389 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670740390 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670740391 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670740391Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670740391Cleaning up temporary data from old regions at 1732670740467 (+76 ms)Region opened successfully at 1732670740485 (+18 ms) 2024-11-27T01:25:40,486 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T01:25:40,520 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697df13e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:25:40,553 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T01:25:40,564 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T01:25:40,564 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T01:25:40,567 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T01:25:40,568 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T01:25:40,574 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T01:25:40,574 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T01:25:40,605 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T01:25:40,614 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T01:25:40,616 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T01:25:40,618 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T01:25:40,620 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T01:25:40,621 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T01:25:40,623 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T01:25:40,627 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T01:25:40,628 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T01:25:40,630 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T01:25:40,632 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T01:25:40,650 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T01:25:40,651 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,658 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7d6f26a4c136,37429,1732670738986, sessionid=0x101570739b00000, setting cluster-up flag (Was=false) 2024-11-27T01:25:40,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,677 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T01:25:40,679 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:40,696 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T01:25:40,697 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:40,703 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T01:25:40,748 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(746): ClusterId : 09523c17-5740-4230-8a95-38d6b0bf24fa 2024-11-27T01:25:40,748 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(746): ClusterId : 09523c17-5740-4230-8a95-38d6b0bf24fa 2024-11-27T01:25:40,750 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(746): ClusterId : 09523c17-5740-4230-8a95-38d6b0bf24fa 2024-11-27T01:25:40,751 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:25:40,751 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:25:40,751 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T01:25:40,756 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:25:40,756 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:25:40,756 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T01:25:40,756 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:25:40,756 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:25:40,756 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T01:25:40,759 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:25:40,760 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:25:40,760 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T01:25:40,760 DEBUG [RS:2;7d6f26a4c136:43859 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac9c465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:25:40,760 DEBUG [RS:0;7d6f26a4c136:41943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b21ca8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:25:40,760 DEBUG [RS:1;7d6f26a4c136:39245 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1563a4fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d6f26a4c136/172.17.0.2:0 2024-11-27T01:25:40,778 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7d6f26a4c136:41943 2024-11-27T01:25:40,782 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:25:40,782 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:25:40,782 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:25:40,786 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7d6f26a4c136:43859 2024-11-27T01:25:40,786 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,37429,1732670738986 with port=41943, startcode=1732670739675 2024-11-27T01:25:40,786 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:25:40,786 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:25:40,786 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:25:40,786 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7d6f26a4c136:39245 2024-11-27T01:25:40,786 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T01:25:40,786 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T01:25:40,787 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T01:25:40,787 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,37429,1732670738986 with port=43859, startcode=1732670739814 2024-11-27T01:25:40,788 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,37429,1732670738986 with port=39245, startcode=1732670739775 2024-11-27T01:25:40,789 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T01:25:40,800 DEBUG [RS:2;7d6f26a4c136:43859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:25:40,800 DEBUG [RS:1;7d6f26a4c136:39245 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:25:40,800 DEBUG [RS:0;7d6f26a4c136:41943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T01:25:40,801 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T01:25:40,808 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T01:25:40,814 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7d6f26a4c136,37429,1732670738986 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T01:25:40,822 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:25:40,823 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:25:40,823 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:25:40,823 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=5, maxPoolSize=5 2024-11-27T01:25:40,823 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7d6f26a4c136:0, corePoolSize=10, maxPoolSize=10 2024-11-27T01:25:40,823 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:40,824 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:25:40,824 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:40,838 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:25:40,839 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T01:25:40,842 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732670770842 2024-11-27T01:25:40,844 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55447, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:25:40,844 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56733, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:25:40,845 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T01:25:40,846 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52997, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T01:25:40,846 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T01:25:40,848 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,848 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T01:25:40,851 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T01:25:40,851 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T01:25:40,852 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T01:25:40,852 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T01:25:40,853 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:25:40,859 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:25:40,860 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T01:25:40,860 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:40,864 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T01:25:40,886 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T01:25:40,886 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T01:25:40,889 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T01:25:40,889 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T01:25:40,891 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670740890,5,FailOnTimeoutGroup] 2024-11-27T01:25:40,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:25:40,892 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670740891,5,FailOnTimeoutGroup] 2024-11-27T01:25:40,892 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:40,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:25:40,892 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T01:25:40,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741831_1007 (size=1321) 2024-11-27T01:25:40,893 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:40,894 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:40,895 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T01:25:40,895 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1 2024-11-27T01:25:40,899 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T01:25:40,899 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T01:25:40,899 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T01:25:40,899 WARN [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T01:25:40,899 WARN [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T01:25:40,899 WARN [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T01:25:40,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:25:40,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:25:40,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741832_1008 (size=32) 2024-11-27T01:25:40,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:40,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:25:40,915 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:25:40,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:40,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:25:40,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:25:40,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:40,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:25:40,923 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:25:40,923 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:40,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:25:40,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:25:40,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:40,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:40,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:25:40,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740 2024-11-27T01:25:40,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740 2024-11-27T01:25:40,932 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:25:40,932 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:25:40,933 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:25:40,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:25:40,939 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:40,940 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60451489, jitterRate=-0.09920261800289154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:25:40,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732670740911Initializing all the Stores at 1732670740913 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670740913Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670740913Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670740913Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670740913Cleaning up temporary data from old regions at 1732670740932 (+19 ms)Region opened successfully at 1732670740942 (+10 ms) 2024-11-27T01:25:40,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:25:40,943 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:25:40,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:25:40,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:25:40,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:25:40,944 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:25:40,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732670740943Disabling compacts and flushes for region at 1732670740943Disabling writes for close at 1732670740943Writing region close event to WAL at 1732670740944 (+1 ms)Closed at 1732670740944 2024-11-27T01:25:40,948 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:25:40,948 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T01:25:40,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T01:25:40,967 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:25:40,970 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T01:25:41,001 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,37429,1732670738986 with port=41943, startcode=1732670739675 2024-11-27T01:25:41,001 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,37429,1732670738986 with port=39245, startcode=1732670739775 2024-11-27T01:25:41,001 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d6f26a4c136,37429,1732670738986 with port=43859, startcode=1732670739814 2024-11-27T01:25:41,003 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:41,005 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:41,013 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,013 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,013 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1 2024-11-27T01:25:41,014 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36155 2024-11-27T01:25:41,014 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:25:41,016 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:41,016 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] master.ServerManager(517): Registering regionserver=7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:41,017 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1 2024-11-27T01:25:41,017 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36155 2024-11-27T01:25:41,017 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:25:41,019 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1 2024-11-27T01:25:41,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:25:41,019 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36155 2024-11-27T01:25:41,019 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T01:25:41,023 DEBUG [RS:0;7d6f26a4c136:41943 {}] zookeeper.ZKUtil(111): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:41,023 WARN [RS:0;7d6f26a4c136:41943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:25:41,023 INFO [RS:0;7d6f26a4c136:41943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:41,023 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:41,024 DEBUG [RS:1;7d6f26a4c136:39245 {}] zookeeper.ZKUtil(111): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,024 WARN [RS:1;7d6f26a4c136:39245 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:25:41,024 INFO [RS:1;7d6f26a4c136:39245 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:41,024 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,024 DEBUG [RS:2;7d6f26a4c136:43859 {}] zookeeper.ZKUtil(111): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:41,024 WARN [RS:2;7d6f26a4c136:43859 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T01:25:41,025 INFO [RS:2;7d6f26a4c136:43859 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:41,025 DEBUG [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:41,026 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,41943,1732670739675] 2024-11-27T01:25:41,026 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,43859,1732670739814] 2024-11-27T01:25:41,026 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d6f26a4c136,39245,1732670739775] 2024-11-27T01:25:41,051 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:25:41,051 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:25:41,051 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T01:25:41,065 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:25:41,065 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:25:41,065 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T01:25:41,071 INFO [RS:0;7d6f26a4c136:41943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:25:41,071 INFO [RS:2;7d6f26a4c136:43859 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:25:41,071 INFO [RS:1;7d6f26a4c136:39245 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:25:41,071 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,071 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,071 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,073 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:25:41,073 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:25:41,073 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T01:25:41,079 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:25:41,079 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:25:41,079 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T01:25:41,081 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,081 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,081 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,081 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,081 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,081 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,082 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:25:41,082 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0, corePoolSize=2, maxPoolSize=2 2024-11-27T01:25:41,082 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:25:41,083 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:0;7d6f26a4c136:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:25:41,083 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,083 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d6f26a4c136:0, corePoolSize=1, maxPoolSize=1 2024-11-27T01:25:41,084 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:25:41,084 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:25:41,084 DEBUG [RS:2;7d6f26a4c136:43859 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:25:41,084 DEBUG [RS:1;7d6f26a4c136:39245 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d6f26a4c136:0, corePoolSize=3, maxPoolSize=3 2024-11-27T01:25:41,085 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,085 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,085 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,085 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,085 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,085 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,41943,1732670739675-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:25:41,087 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,087 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,087 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,087 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,087 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,39245,1732670739775-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,088 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43859,1732670739814-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:25:41,113 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:25:41,113 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:25:41,114 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T01:25:41,115 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,41943,1732670739675-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,115 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,39245,1732670739775-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,115 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,43859,1732670739814-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,115 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,115 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,115 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,116 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.Replication(171): 7d6f26a4c136,41943,1732670739675 started 2024-11-27T01:25:41,116 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.Replication(171): 7d6f26a4c136,39245,1732670739775 started 2024-11-27T01:25:41,116 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.Replication(171): 7d6f26a4c136,43859,1732670739814 started 2024-11-27T01:25:41,121 WARN [7d6f26a4c136:37429 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T01:25:41,143 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,143 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,143 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,144 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,39245,1732670739775, RpcServer on 7d6f26a4c136/172.17.0.2:39245, sessionid=0x101570739b00002 2024-11-27T01:25:41,144 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,41943,1732670739675, RpcServer on 7d6f26a4c136/172.17.0.2:41943, sessionid=0x101570739b00001 2024-11-27T01:25:41,144 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(1482): Serving as 7d6f26a4c136,43859,1732670739814, RpcServer on 7d6f26a4c136/172.17.0.2:43859, sessionid=0x101570739b00003 2024-11-27T01:25:41,145 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:25:41,145 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:25:41,145 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T01:25:41,145 DEBUG [RS:2;7d6f26a4c136:43859 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:41,145 DEBUG [RS:0;7d6f26a4c136:41943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:41,145 DEBUG [RS:1;7d6f26a4c136:39245 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,145 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,43859,1732670739814' 2024-11-27T01:25:41,145 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,39245,1732670739775' 2024-11-27T01:25:41,145 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,41943,1732670739675' 2024-11-27T01:25:41,145 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:25:41,145 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:25:41,145 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T01:25:41,146 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:25:41,146 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:25:41,147 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T01:25:41,147 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:25:41,147 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:25:41,147 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T01:25:41,147 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:25:41,147 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:25:41,147 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T01:25:41,147 DEBUG [RS:2;7d6f26a4c136:43859 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:41,147 DEBUG [RS:0;7d6f26a4c136:41943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:41,147 DEBUG [RS:1;7d6f26a4c136:39245 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,148 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,43859,1732670739814' 2024-11-27T01:25:41,148 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,39245,1732670739775' 2024-11-27T01:25:41,148 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d6f26a4c136,41943,1732670739675' 2024-11-27T01:25:41,148 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:25:41,148 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:25:41,148 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T01:25:41,148 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:25:41,148 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:25:41,149 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T01:25:41,149 DEBUG [RS:2;7d6f26a4c136:43859 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:25:41,149 INFO [RS:2;7d6f26a4c136:43859 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:25:41,149 DEBUG [RS:0;7d6f26a4c136:41943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:25:41,149 DEBUG [RS:1;7d6f26a4c136:39245 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T01:25:41,149 INFO [RS:0;7d6f26a4c136:41943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:25:41,149 INFO [RS:2;7d6f26a4c136:43859 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:25:41,149 INFO [RS:1;7d6f26a4c136:39245 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T01:25:41,150 INFO [RS:0;7d6f26a4c136:41943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:25:41,150 INFO [RS:1;7d6f26a4c136:39245 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T01:25:41,255 INFO [RS:2;7d6f26a4c136:43859 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:41,255 INFO [RS:0;7d6f26a4c136:41943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:41,255 INFO [RS:1;7d6f26a4c136:39245 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:41,259 INFO [RS:2;7d6f26a4c136:43859 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C43859%2C1732670739814, suffix=, logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,43859,1732670739814, archiveDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs, maxLogs=32 2024-11-27T01:25:41,259 INFO [RS:0;7d6f26a4c136:41943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C41943%2C1732670739675, suffix=, logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675, archiveDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs, maxLogs=32 2024-11-27T01:25:41,259 INFO [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C39245%2C1732670739775, suffix=, logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775, archiveDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs, maxLogs=32 2024-11-27T01:25:41,279 DEBUG [RS:1;7d6f26a4c136:39245 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775/7d6f26a4c136%2C39245%2C1732670739775.1732670741263, exclude list is [], retry=0 2024-11-27T01:25:41,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:41,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:41,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:41,285 DEBUG [RS:2;7d6f26a4c136:43859 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,43859,1732670739814/7d6f26a4c136%2C43859%2C1732670739814.1732670741264, exclude list is [], retry=0 2024-11-27T01:25:41,285 DEBUG [RS:0;7d6f26a4c136:41943 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675/7d6f26a4c136%2C41943%2C1732670739675.1732670741264, exclude list is [], retry=0 2024-11-27T01:25:41,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:41,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:41,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:41,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:41,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:41,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:41,318 INFO [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775/7d6f26a4c136%2C39245%2C1732670739775.1732670741263 2024-11-27T01:25:41,319 DEBUG [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:41,335 INFO [RS:2;7d6f26a4c136:43859 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,43859,1732670739814/7d6f26a4c136%2C43859%2C1732670739814.1732670741264 2024-11-27T01:25:41,336 DEBUG [RS:2;7d6f26a4c136:43859 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:41,340 INFO [RS:0;7d6f26a4c136:41943 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 2024-11-27T01:25:41,341 DEBUG [RS:0;7d6f26a4c136:41943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:41,374 DEBUG [7d6f26a4c136:37429 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-27T01:25:41,382 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:25:41,388 INFO [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:25:41,388 INFO [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:25:41,388 INFO [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:25:41,388 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:25:41,396 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,401 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,39245,1732670739775, state=OPENING 2024-11-27T01:25:41,407 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T01:25:41,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:41,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:41,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:41,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:25:41,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,411 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T01:25:41,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,39245,1732670739775}] 2024-11-27T01:25:41,587 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:25:41,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:25:41,603 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T01:25:41,603 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:41,604 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T01:25:41,607 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d6f26a4c136%2C39245%2C1732670739775.meta, suffix=.meta, logDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775, archiveDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs, maxLogs=32 2024-11-27T01:25:41,623 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775/7d6f26a4c136%2C39245%2C1732670739775.meta.1732670741608.meta, exclude list is [], retry=0 2024-11-27T01:25:41,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:41,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:41,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:41,659 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,39245,1732670739775/7d6f26a4c136%2C39245%2C1732670739775.meta.1732670741608.meta 2024-11-27T01:25:41,659 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:41,659 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:41,661 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T01:25:41,664 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T01:25:41,669 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T01:25:41,673 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T01:25:41,673 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:41,674 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T01:25:41,674 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T01:25:41,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T01:25:41,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T01:25:41,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:41,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:41,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T01:25:41,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T01:25:41,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:41,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:41,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T01:25:41,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T01:25:41,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:41,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:41,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T01:25:41,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T01:25:41,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:41,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T01:25:41,685 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T01:25:41,687 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740 2024-11-27T01:25:41,689 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740 2024-11-27T01:25:41,691 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T01:25:41,691 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T01:25:41,692 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T01:25:41,694 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T01:25:41,695 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69734642, jitterRate=0.03912714123725891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T01:25:41,696 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T01:25:41,697 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732670741674Writing region info on filesystem at 1732670741674Initializing all the Stores at 1732670741676 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670741676Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670741676Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670741676Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732670741676Cleaning up temporary data from old regions at 1732670741691 (+15 ms)Running coprocessor post-open hooks at 1732670741696 (+5 ms)Region opened successfully at 1732670741697 (+1 ms) 2024-11-27T01:25:41,704 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732670741578 2024-11-27T01:25:41,715 DEBUG [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T01:25:41,716 INFO [RS_OPEN_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T01:25:41,718 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,719 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d6f26a4c136,39245,1732670739775, state=OPEN 2024-11-27T01:25:41,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:25:41,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:25:41,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:25:41,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T01:25:41,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T01:25:41,722 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:41,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T01:25:41,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d6f26a4c136,39245,1732670739775 in 311 msec 2024-11-27T01:25:41,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T01:25:41,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 774 msec 2024-11-27T01:25:41,735 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T01:25:41,735 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T01:25:41,756 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:25:41,757 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,39245,1732670739775, seqNum=-1] 2024-11-27T01:25:41,784 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:25:41,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:25:41,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0720 sec 2024-11-27T01:25:41,805 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732670741805, completionTime=-1 2024-11-27T01:25:41,808 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-27T01:25:41,808 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T01:25:41,836 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-27T01:25:41,836 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732670801836 2024-11-27T01:25:41,836 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732670861836 2024-11-27T01:25:41,836 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-27T01:25:41,838 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T01:25:41,846 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,37429,1732670738986-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,846 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,37429,1732670738986-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,846 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,37429,1732670738986-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,849 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7d6f26a4c136:37429, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,849 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,850 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,857 DEBUG [master/7d6f26a4c136:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T01:25:41,910 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.002sec 2024-11-27T01:25:41,912 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T01:25:41,913 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T01:25:41,914 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T01:25:41,915 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T01:25:41,915 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T01:25:41,916 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,37429,1732670738986-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T01:25:41,921 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,37429,1732670738986-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T01:25:41,926 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T01:25:41,927 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T01:25:41,927 INFO [master/7d6f26a4c136:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d6f26a4c136,37429,1732670738986-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:41,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692dfaac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:25:41,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d6f26a4c136,37429,-1 for getting cluster id 2024-11-27T01:25:41,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T01:25:41,970 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09523c17-5740-4230-8a95-38d6b0bf24fa' 2024-11-27T01:25:41,972 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T01:25:41,973 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09523c17-5740-4230-8a95-38d6b0bf24fa" 2024-11-27T01:25:41,973 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26198064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:25:41,973 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d6f26a4c136,37429,-1] 2024-11-27T01:25:41,976 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T01:25:41,978 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:25:41,978 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T01:25:41,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b654e0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T01:25:41,982 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T01:25:41,990 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d6f26a4c136,39245,1732670739775, seqNum=-1] 2024-11-27T01:25:41,990 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:25:41,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:25:42,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:42,012 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:36155/hbase 2024-11-27T01:25:42,027 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6280 2024-11-27T01:25:42,046 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:42,049 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:42,050 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:42,054 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-09730633, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-09730633, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:42,077 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-09730633/hregion-09730633.1732670742056, exclude list is [], retry=0 2024-11-27T01:25:42,082 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:42,082 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:42,083 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:42,115 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-09730633/hregion-09730633.1732670742056 2024-11-27T01:25:42,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:42,116 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6f47c7142a9186f0798f41ca31c80ec0, NAME => 'testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:25:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741838_1014 (size=64) 2024-11-27T01:25:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741838_1014 (size=64) 2024-11-27T01:25:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741838_1014 (size=64) 2024-11-27T01:25:42,130 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:42,132 INFO [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,134 INFO [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f47c7142a9186f0798f41ca31c80ec0 columnFamilyName a 2024-11-27T01:25:42,134 DEBUG [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:42,135 INFO [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] regionserver.HStore(327): Store=6f47c7142a9186f0798f41ca31c80ec0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:42,135 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,137 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,137 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,138 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,138 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,141 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,145 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:42,146 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6f47c7142a9186f0798f41ca31c80ec0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64740308, jitterRate=-0.03529423475265503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:25:42,147 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6f47c7142a9186f0798f41ca31c80ec0: Writing region info on filesystem at 1732670742130Initializing all the Stores at 1732670742132 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670742132Cleaning up temporary data from old regions at 1732670742138 (+6 ms)Region opened successfully at 1732670742147 (+9 ms) 2024-11-27T01:25:42,147 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6f47c7142a9186f0798f41ca31c80ec0, disabling compactions & flushes 2024-11-27T01:25:42,147 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0. 2024-11-27T01:25:42,147 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0. 2024-11-27T01:25:42,147 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0. after waiting 0 ms 2024-11-27T01:25:42,148 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0. 2024-11-27T01:25:42,148 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0. 2024-11-27T01:25:42,148 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6f47c7142a9186f0798f41ca31c80ec0: Waiting for close lock at 1732670742147Disabling compacts and flushes for region at 1732670742147Disabling writes for close at 1732670742147Writing region close event to WAL at 1732670742148 (+1 ms)Closed at 1732670742148 2024-11-27T01:25:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741837_1013 (size=175) 2024-11-27T01:25:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741837_1013 (size=175) 2024-11-27T01:25:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741837_1013 (size=175) 2024-11-27T01:25:42,159 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:42,159 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-09730633:(num 1732670742056) 2024-11-27T01:25:42,161 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:25:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741839_1015 (size=469) 2024-11-27T01:25:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741839_1015 (size=469) 2024-11-27T01:25:42,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741839_1015 (size=469) 2024-11-27T01:25:42,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:25:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741840_1016 (size=322) 2024-11-27T01:25:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741840_1016 (size=322) 2024-11-27T01:25:42,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741840_1016 (size=322) 2024-11-27T01:25:42,277 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1, size=469 (469bytes) 2024-11-27T01:25:42,278 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-27T01:25:42,278 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-27T01:25:42,279 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1 2024-11-27T01:25:42,284 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1 after 3ms 2024-11-27T01:25:42,316 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:42,317 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1 took 41ms 2024-11-27T01:25:42,322 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1 so closing down 2024-11-27T01:25:42,322 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:42,325 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T01:25:42,351 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T01:25:42,352 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:42,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741841_1017 (size=469) 2024-11-27T01:25:42,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741841_1017 (size=469) 2024-11-27T01:25:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741841_1017 (size=469) 2024-11-27T01:25:42,364 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T01:25:42,367 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002 2024-11-27T01:25:42,370 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-27T01:25:42,371 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1, journal: Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1, size=469 (469bytes) at 1732670742277Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1 so closing down at 1732670742322 (+45 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000001-wal-1.temp at 1732670742351 (+29 ms)3 split writer threads finished at 1732670742352 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732670742364 (+12 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002 at 1732670742367 (+3 ms)Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732670742370 (+3 ms) 2024-11-27T01:25:42,385 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2, size=322 (322bytes) 2024-11-27T01:25:42,385 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2 2024-11-27T01:25:42,386 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2 after 1ms 2024-11-27T01:25:42,418 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:42,418 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2 took 33ms 2024-11-27T01:25:42,421 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2 so closing down 2024-11-27T01:25:42,421 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:42,425 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T01:25:42,462 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T01:25:42,462 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:42,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741842_1018 (size=322) 2024-11-27T01:25:42,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741842_1018 (size=322) 2024-11-27T01:25:42,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741842_1018 (size=322) 2024-11-27T01:25:42,479 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T01:25:42,514 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:42,517 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-27T01:25:42,519 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-27T01:25:42,520 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2, journal: Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2, size=322 (322bytes) at 1732670742385Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2 so closing down at 1732670742421 (+36 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002-wal-2.temp at 1732670742462 (+41 ms)3 split writer threads finished at 1732670742462Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732670742479 (+17 ms)Processed 1 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732670742519 (+40 ms) 2024-11-27T01:25:42,520 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:42,522 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:42,536 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal.1732670742523, exclude list is [], retry=0 2024-11-27T01:25:42,541 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:42,541 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:42,542 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:42,570 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal.1732670742523 2024-11-27T01:25:42,570 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:42,570 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f47c7142a9186f0798f41ca31c80ec0, NAME => 'testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:42,571 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:42,571 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,571 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,573 INFO [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,575 INFO [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f47c7142a9186f0798f41ca31c80ec0 columnFamilyName a 2024-11-27T01:25:42,575 DEBUG [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:42,576 INFO [StoreOpener-6f47c7142a9186f0798f41ca31c80ec0-1 {}] regionserver.HStore(327): Store=6f47c7142a9186f0798f41ca31c80ec0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:42,576 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,577 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,580 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,582 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002 2024-11-27T01:25:42,613 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:42,621 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002 2024-11-27T01:25:42,625 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6f47c7142a9186f0798f41ca31c80ec0 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T01:25:42,684 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/.tmp/a/7e74d64b676b4b77a2a25d0ba8b9a6f8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732670742160/Put/seqid=0 2024-11-27T01:25:42,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741844_1020 (size=5170) 2024-11-27T01:25:42,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741844_1020 (size=5170) 2024-11-27T01:25:42,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741844_1020 (size=5170) 2024-11-27T01:25:42,700 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/.tmp/a/7e74d64b676b4b77a2a25d0ba8b9a6f8 2024-11-27T01:25:42,751 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/.tmp/a/7e74d64b676b4b77a2a25d0ba8b9a6f8 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/a/7e74d64b676b4b77a2a25d0ba8b9a6f8 2024-11-27T01:25:42,762 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/a/7e74d64b676b4b77a2a25d0ba8b9a6f8, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T01:25:42,767 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6f47c7142a9186f0798f41ca31c80ec0 in 143ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T01:25:42,769 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/0000000000000000002 2024-11-27T01:25:42,770 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,770 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6f47c7142a9186f0798f41ca31c80ec0 2024-11-27T01:25:42,776 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/6f47c7142a9186f0798f41ca31c80ec0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T01:25:42,778 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6f47c7142a9186f0798f41ca31c80ec0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73842634, jitterRate=0.10034099221229553}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:25:42,779 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6f47c7142a9186f0798f41ca31c80ec0: Writing region info on filesystem at 1732670742571Initializing all the Stores at 1732670742573 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670742573Obtaining lock to block concurrent updates at 1732670742625 (+52 ms)Preparing flush snapshotting stores in 6f47c7142a9186f0798f41ca31c80ec0 at 1732670742625Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732670742629 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732670742047.6f47c7142a9186f0798f41ca31c80ec0. at 1732670742629Flushing 6f47c7142a9186f0798f41ca31c80ec0/a: creating writer at 1732670742631 (+2 ms)Flushing 6f47c7142a9186f0798f41ca31c80ec0/a: appending metadata at 1732670742672 (+41 ms)Flushing 6f47c7142a9186f0798f41ca31c80ec0/a: closing flushed file at 1732670742676 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3204d41d: reopening flushed file at 1732670742749 (+73 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6f47c7142a9186f0798f41ca31c80ec0 in 143ms, sequenceid=2, compaction requested=false; wal=null at 1732670742768 (+19 ms)Cleaning up temporary data from old regions at 1732670742770 (+2 ms)Region opened successfully at 1732670742778 (+8 ms) 2024-11-27T01:25:42,807 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:53582 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57552 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:33784 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:48174 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:36155/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:53820 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:47434 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 275), ProcessCount=11 (was 11), AvailableMemoryMB=6255 (was 6280) 2024-11-27T01:25:42,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6252 2024-11-27T01:25:42,840 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:42,842 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:42,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:42,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95813950, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-95813950, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:42,862 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95813950/hregion-95813950.1732670742848, exclude list is [], retry=0 2024-11-27T01:25:42,866 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:42,867 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:42,868 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:42,903 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95813950/hregion-95813950.1732670742848 2024-11-27T01:25:42,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:42,904 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a30bde320c61574f860ed281204ee2c1, NAME => 'testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:25:42,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741846_1022 (size=64) 2024-11-27T01:25:42,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741846_1022 (size=64) 2024-11-27T01:25:42,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741846_1022 (size=64) 2024-11-27T01:25:42,943 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:42,947 INFO [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,950 INFO [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30bde320c61574f860ed281204ee2c1 columnFamilyName a 2024-11-27T01:25:42,950 DEBUG [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:42,951 INFO [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] regionserver.HStore(327): Store=a30bde320c61574f860ed281204ee2c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:42,952 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,954 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,954 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,957 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:42,961 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:42,962 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a30bde320c61574f860ed281204ee2c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61099577, jitterRate=-0.0895453542470932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:25:42,962 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a30bde320c61574f860ed281204ee2c1: Writing region info on filesystem at 1732670742943Initializing all the Stores at 1732670742945 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670742945Cleaning up temporary data from old regions at 1732670742954 (+9 ms)Region opened successfully at 1732670742962 (+8 ms) 2024-11-27T01:25:42,962 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a30bde320c61574f860ed281204ee2c1, disabling compactions & flushes 2024-11-27T01:25:42,962 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1. 2024-11-27T01:25:42,962 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1. 2024-11-27T01:25:42,962 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1. after waiting 0 ms 2024-11-27T01:25:42,962 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1. 2024-11-27T01:25:42,963 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1. 2024-11-27T01:25:42,963 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a30bde320c61574f860ed281204ee2c1: Waiting for close lock at 1732670742962Disabling compacts and flushes for region at 1732670742962Disabling writes for close at 1732670742962Writing region close event to WAL at 1732670742963 (+1 ms)Closed at 1732670742963 2024-11-27T01:25:42,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741845_1021 (size=175) 2024-11-27T01:25:42,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741845_1021 (size=175) 2024-11-27T01:25:42,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741845_1021 (size=175) 2024-11-27T01:25:42,972 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:42,972 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95813950:(num 1732670742848) 2024-11-27T01:25:42,973 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:25:43,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741847_1023 (size=469) 2024-11-27T01:25:43,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741847_1023 (size=469) 2024-11-27T01:25:43,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741847_1023 (size=469) 2024-11-27T01:25:43,027 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-27T01:25:43,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741848_1024 (size=322) 2024-11-27T01:25:43,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741848_1024 (size=322) 2024-11-27T01:25:43,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741848_1024 (size=322) 2024-11-27T01:25:43,082 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2, size=322 (322bytes) 2024-11-27T01:25:43,082 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2 2024-11-27T01:25:43,083 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2 after 1ms 2024-11-27T01:25:43,111 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:43,112 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2 took 30ms 2024-11-27T01:25:43,114 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2 so closing down 2024-11-27T01:25:43,114 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:43,116 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-27T01:25:43,143 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002-wal-2.temp 2024-11-27T01:25:43,143 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:43,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741849_1025 (size=322) 2024-11-27T01:25:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741849_1025 (size=322) 2024-11-27T01:25:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741849_1025 (size=322) 2024-11-27T01:25:43,153 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T01:25:43,154 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 2024-11-27T01:25:43,155 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-27T01:25:43,155 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2, journal: Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2, size=322 (322bytes) at 1732670743082Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2 so closing down at 1732670743114 (+32 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002-wal-2.temp at 1732670743143 (+29 ms)3 split writer threads finished at 1732670743143Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732670743153 (+10 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 at 1732670743154 (+1 ms)Processed 1 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732670743155 (+1 ms) 2024-11-27T01:25:43,169 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1, size=469 (469bytes) 2024-11-27T01:25:43,169 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1 2024-11-27T01:25:43,169 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1 after 0ms 2024-11-27T01:25:43,197 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:43,197 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1 took 28ms 2024-11-27T01:25:43,199 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1 so closing down 2024-11-27T01:25:43,199 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:43,201 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-27T01:25:43,228 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000001-wal-1.temp 2024-11-27T01:25:43,229 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:43,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741850_1026 (size=469) 2024-11-27T01:25:43,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741850_1026 (size=469) 2024-11-27T01:25:43,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741850_1026 (size=469) 2024-11-27T01:25:43,639 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T01:25:43,670 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:43,672 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002, length=322 2024-11-27T01:25:43,675 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 2024-11-27T01:25:43,675 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 478 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-27T01:25:43,675 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1, journal: Splitting hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1, size=469 (469bytes) at 1732670743169Finishing writing output for hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1 so closing down at 1732670743199 (+30 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000001-wal-1.temp at 1732670743228 (+29 ms)3 split writer threads finished at 1732670743229 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732670743639 (+410 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 at 1732670743675 (+36 ms)Processed 2 edits across 1 Regions in 478 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732670743675 2024-11-27T01:25:43,675 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:43,678 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:43,693 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal.1732670743679, exclude list is [], retry=0 2024-11-27T01:25:43,696 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:43,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:43,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:43,724 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal.1732670743679 2024-11-27T01:25:43,724 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:43,724 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a30bde320c61574f860ed281204ee2c1, NAME => 'testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:43,724 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:43,725 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,725 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,727 INFO [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,728 INFO [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a30bde320c61574f860ed281204ee2c1 columnFamilyName a 2024-11-27T01:25:43,728 DEBUG [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:43,729 INFO [StoreOpener-a30bde320c61574f860ed281204ee2c1-1 {}] regionserver.HStore(327): Store=a30bde320c61574f860ed281204ee2c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:43,729 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,730 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,732 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,734 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 2024-11-27T01:25:43,764 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:43,766 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 2024-11-27T01:25:43,766 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a30bde320c61574f860ed281204ee2c1 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-27T01:25:43,782 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/.tmp/a/efb7544bbf644091a035caad11bc666e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732670742972/Put/seqid=0 2024-11-27T01:25:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741852_1028 (size=5170) 2024-11-27T01:25:43,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741852_1028 (size=5170) 2024-11-27T01:25:43,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741852_1028 (size=5170) 2024-11-27T01:25:43,791 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/.tmp/a/efb7544bbf644091a035caad11bc666e 2024-11-27T01:25:43,799 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/.tmp/a/efb7544bbf644091a035caad11bc666e as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/a/efb7544bbf644091a035caad11bc666e 2024-11-27T01:25:43,806 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/a/efb7544bbf644091a035caad11bc666e, entries=2, sequenceid=2, filesize=5.0 K 2024-11-27T01:25:43,807 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a30bde320c61574f860ed281204ee2c1 in 41ms, sequenceid=2, compaction requested=false; wal=null 2024-11-27T01:25:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/0000000000000000002 2024-11-27T01:25:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,808 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,810 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a30bde320c61574f860ed281204ee2c1 2024-11-27T01:25:43,813 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/a30bde320c61574f860ed281204ee2c1/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-27T01:25:43,814 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a30bde320c61574f860ed281204ee2c1; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66954002, jitterRate=-0.0023076236248016357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:25:43,815 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a30bde320c61574f860ed281204ee2c1: Writing region info on filesystem at 1732670743725Initializing all the Stores at 1732670743726 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670743726Obtaining lock to block concurrent updates at 1732670743766 (+40 ms)Preparing flush snapshotting stores in a30bde320c61574f860ed281204ee2c1 at 1732670743766Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732670743766Flushing stores of testReplayEditsWrittenIntoWAL,,1732670742841.a30bde320c61574f860ed281204ee2c1. at 1732670743766Flushing a30bde320c61574f860ed281204ee2c1/a: creating writer at 1732670743767 (+1 ms)Flushing a30bde320c61574f860ed281204ee2c1/a: appending metadata at 1732670743781 (+14 ms)Flushing a30bde320c61574f860ed281204ee2c1/a: closing flushed file at 1732670743781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@792f7dec: reopening flushed file at 1732670743798 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a30bde320c61574f860ed281204ee2c1 in 41ms, sequenceid=2, compaction requested=false; wal=null at 1732670743807 (+9 ms)Cleaning up temporary data from old regions at 1732670743808 (+1 ms)Region opened successfully at 1732670743814 (+6 ms) 2024-11-27T01:25:43,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:53582 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57628 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:53820 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:47512 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:33784 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:48256 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=275 (was 275), ProcessCount=11 (was 11), AvailableMemoryMB=6235 (was 6252) 2024-11-27T01:25:43,848 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=275, ProcessCount=11, AvailableMemoryMB=6234 2024-11-27T01:25:43,863 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:43,865 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:43,866 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:43,869 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44366849, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-44366849, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:43,882 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44366849/hregion-44366849.1732670743869, exclude list is [], retry=0 2024-11-27T01:25:43,885 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:43,886 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:43,886 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:43,913 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44366849/hregion-44366849.1732670743869 2024-11-27T01:25:43,914 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:43,914 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2bcce029151ca4a5343818abe3b79c28, NAME => 'testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:25:43,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741854_1030 (size=64) 2024-11-27T01:25:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741854_1030 (size=64) 2024-11-27T01:25:43,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741854_1030 (size=64) 2024-11-27T01:25:43,928 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:43,929 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,931 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bcce029151ca4a5343818abe3b79c28 columnFamilyName a 2024-11-27T01:25:43,931 DEBUG [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:43,932 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(327): Store=2bcce029151ca4a5343818abe3b79c28/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:43,932 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,933 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bcce029151ca4a5343818abe3b79c28 columnFamilyName b 2024-11-27T01:25:43,933 DEBUG [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:43,934 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(327): Store=2bcce029151ca4a5343818abe3b79c28/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:43,934 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,936 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bcce029151ca4a5343818abe3b79c28 columnFamilyName c 2024-11-27T01:25:43,936 DEBUG [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:43,936 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(327): Store=2bcce029151ca4a5343818abe3b79c28/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:43,936 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,937 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,938 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,939 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,939 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,939 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:43,941 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:43,943 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:43,944 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2bcce029151ca4a5343818abe3b79c28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62204133, jitterRate=-0.07308618724346161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:43,944 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2bcce029151ca4a5343818abe3b79c28: Writing region info on filesystem at 1732670743928Initializing all the Stores at 1732670743929 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670743929Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670743929Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670743929Cleaning up temporary data from old regions at 1732670743939 (+10 ms)Region opened successfully at 1732670743944 (+5 ms) 2024-11-27T01:25:43,944 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2bcce029151ca4a5343818abe3b79c28, disabling compactions & flushes 2024-11-27T01:25:43,944 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:43,944 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:43,944 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. after waiting 0 ms 2024-11-27T01:25:43,945 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:43,945 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:43,945 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2bcce029151ca4a5343818abe3b79c28: Waiting for close lock at 1732670743944Disabling compacts and flushes for region at 1732670743944Disabling writes for close at 1732670743945 (+1 ms)Writing region close event to WAL at 1732670743945Closed at 1732670743945 2024-11-27T01:25:43,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741853_1029 (size=175) 2024-11-27T01:25:43,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741853_1029 (size=175) 2024-11-27T01:25:43,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741853_1029 (size=175) 2024-11-27T01:25:43,954 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:43,954 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44366849:(num 1732670743869) 2024-11-27T01:25:43,954 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:43,957 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:43,971 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957, exclude list is [], retry=0 2024-11-27T01:25:43,974 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:43,974 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:43,975 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:44,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 2024-11-27T01:25:44,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:44,242 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957, size=0 (0bytes) 2024-11-27T01:25:44,242 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 might be still open, length is 0 2024-11-27T01:25:44,242 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 2024-11-27T01:25:44,244 WARN [IPC Server handler 4 on default port 36155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-27T01:25:44,244 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 after 2ms 2024-11-27T01:25:46,962 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57662 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45665:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57662 dst: /127.0.0.1:45665 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45665 remote=/127.0.0.1:57662]. Total timeout mills is 60000, 57240 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:46,963 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:47544 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47544 dst: /127.0.0.1:44005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:46,963 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:48304 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48304 dst: /127.0.0.1:39261 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741855_1032 (size=497775) 2024-11-27T01:25:46,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741855_1032 (size=497775) 2024-11-27T01:25:46,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741855_1032 (size=497775) 2024-11-27T01:25:47,177 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:25:47,226 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T01:25:48,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 after 4003ms 2024-11-27T01:25:48,276 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:48,278 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 took 4036ms 2024-11-27T01:25:48,283 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732670743957.temp 2024-11-27T01:25:48,313 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000000001-wal.1732670743957.temp 2024-11-27T01:25:48,364 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957; continuing. 2024-11-27T01:25:48,364 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 so closing down 2024-11-27T01:25:48,364 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:48,365 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:48,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741856_1033 (size=497783) 2024-11-27T01:25:48,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741856_1033 (size=497783) 2024-11-27T01:25:48,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741856_1033 (size=497783) 2024-11-27T01:25:48,369 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000000001-wal.1732670743957.temp (wrote 3002 edits, skipped 0 edits in 40 ms) 2024-11-27T01:25:48,371 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000000001-wal.1732670743957.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000003002 2024-11-27T01:25:48,371 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 93 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:25:48,371 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957, journal: Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957, size=0 (0bytes) at 1732670744242Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000000001-wal.1732670743957.temp at 1732670748313 (+4071 ms)Split 1024 edits, skipped 0 edits. at 1732670748322 (+9 ms)Split 2048 edits, skipped 0 edits. at 1732670748347 (+25 ms)Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 so closing down at 1732670748364 (+17 ms)3 split writer threads finished at 1732670748365 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000000001-wal.1732670743957.temp (wrote 3002 edits, skipped 0 edits in 40 ms) at 1732670748369 (+4 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000000001-wal.1732670743957.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000003002 at 1732670748371 (+2 ms)Processed 3002 edits across 1 Regions in 93 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957, size=0, length=0, corrupted=false, cancelled=false at 1732670748371 2024-11-27T01:25:48,373 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670743957 2024-11-27T01:25:48,374 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000003002 2024-11-27T01:25:48,374 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:48,376 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:48,390 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670748377, exclude list is [], retry=0 2024-11-27T01:25:48,393 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:48,394 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:48,394 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:48,424 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670748377 2024-11-27T01:25:48,424 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:48,425 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:48,426 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:48,428 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bcce029151ca4a5343818abe3b79c28 columnFamilyName a 2024-11-27T01:25:48,428 DEBUG [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:48,429 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(327): Store=2bcce029151ca4a5343818abe3b79c28/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:48,429 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:48,430 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bcce029151ca4a5343818abe3b79c28 columnFamilyName b 2024-11-27T01:25:48,430 DEBUG [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:48,431 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(327): Store=2bcce029151ca4a5343818abe3b79c28/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:48,431 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:48,432 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2bcce029151ca4a5343818abe3b79c28 columnFamilyName c 2024-11-27T01:25:48,432 DEBUG [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:48,433 INFO [StoreOpener-2bcce029151ca4a5343818abe3b79c28-1 {}] regionserver.HStore(327): Store=2bcce029151ca4a5343818abe3b79c28/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:48,433 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:48,434 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:48,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:48,437 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000003002 2024-11-27T01:25:48,475 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:48,513 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:25:48,899 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2bcce029151ca4a5343818abe3b79c28 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T01:25:48,947 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/a/3508272d2d0647f39c1b84dc9ce7bb72 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732670744010/Put/seqid=0 2024-11-27T01:25:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741858_1035 (size=50463) 2024-11-27T01:25:48,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741858_1035 (size=50463) 2024-11-27T01:25:48,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741858_1035 (size=50463) 2024-11-27T01:25:48,961 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/a/3508272d2d0647f39c1b84dc9ce7bb72 2024-11-27T01:25:48,971 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/a/3508272d2d0647f39c1b84dc9ce7bb72 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/a/3508272d2d0647f39c1b84dc9ce7bb72 2024-11-27T01:25:48,979 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/a/3508272d2d0647f39c1b84dc9ce7bb72, entries=754, sequenceid=754, filesize=49.3 K 2024-11-27T01:25:48,980 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 2bcce029151ca4a5343818abe3b79c28 in 81ms, sequenceid=754, compaction requested=false; wal=null 2024-11-27T01:25:48,996 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:25:48,997 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2bcce029151ca4a5343818abe3b79c28 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T01:25:49,006 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/a/dad4c063a942459c8c1662873d221b7f is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732670744065/Put/seqid=0 2024-11-27T01:25:49,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741859_1036 (size=20072) 2024-11-27T01:25:49,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741859_1036 (size=20072) 2024-11-27T01:25:49,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741859_1036 (size=20072) 2024-11-27T01:25:49,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/a/dad4c063a942459c8c1662873d221b7f 2024-11-27T01:25:49,042 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/b/4028a76773144620bdb43dd39bab2768 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732670744091/Put/seqid=0 2024-11-27T01:25:49,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741860_1037 (size=35835) 2024-11-27T01:25:49,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741860_1037 (size=35835) 2024-11-27T01:25:49,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741860_1037 (size=35835) 2024-11-27T01:25:49,053 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/b/4028a76773144620bdb43dd39bab2768 2024-11-27T01:25:49,060 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/a/dad4c063a942459c8c1662873d221b7f as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/a/dad4c063a942459c8c1662873d221b7f 2024-11-27T01:25:49,067 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/a/dad4c063a942459c8c1662873d221b7f, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-27T01:25:49,070 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/b/4028a76773144620bdb43dd39bab2768 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/b/4028a76773144620bdb43dd39bab2768 2024-11-27T01:25:49,076 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/b/4028a76773144620bdb43dd39bab2768, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-27T01:25:49,077 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 2bcce029151ca4a5343818abe3b79c28 in 81ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-27T01:25:49,094 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:25:49,094 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2bcce029151ca4a5343818abe3b79c28 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-27T01:25:49,103 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/b/b94bfe7e698049de87ba8a5bc4bcb562 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732670744116/Put/seqid=0 2024-11-27T01:25:49,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741861_1038 (size=35082) 2024-11-27T01:25:49,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741861_1038 (size=35082) 2024-11-27T01:25:49,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741861_1038 (size=35082) 2024-11-27T01:25:49,114 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/b/b94bfe7e698049de87ba8a5bc4bcb562 2024-11-27T01:25:49,139 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/c/be7ec2818d954883b19af5fa58273dae is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732670744153/Put/seqid=0 2024-11-27T01:25:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741862_1039 (size=20825) 2024-11-27T01:25:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741862_1039 (size=20825) 2024-11-27T01:25:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741862_1039 (size=20825) 2024-11-27T01:25:49,148 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/c/be7ec2818d954883b19af5fa58273dae 2024-11-27T01:25:49,155 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/b/b94bfe7e698049de87ba8a5bc4bcb562 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/b/b94bfe7e698049de87ba8a5bc4bcb562 2024-11-27T01:25:49,161 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/b/b94bfe7e698049de87ba8a5bc4bcb562, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-27T01:25:49,163 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/c/be7ec2818d954883b19af5fa58273dae as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/c/be7ec2818d954883b19af5fa58273dae 2024-11-27T01:25:49,168 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/c/be7ec2818d954883b19af5fa58273dae, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-27T01:25:49,169 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 2bcce029151ca4a5343818abe3b79c28 in 75ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-27T01:25:49,179 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732670744200/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:49,182 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000003002 2024-11-27T01:25:49,182 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-27T01:25:49,182 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2bcce029151ca4a5343818abe3b79c28 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-27T01:25:49,190 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/c/90e421216c854ac3a9ae7559f8554d0a is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732670744161/Put/seqid=0 2024-11-27T01:25:49,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741863_1040 (size=50301) 2024-11-27T01:25:49,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741863_1040 (size=50301) 2024-11-27T01:25:49,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741863_1040 (size=50301) 2024-11-27T01:25:49,200 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/c/90e421216c854ac3a9ae7559f8554d0a 2024-11-27T01:25:49,206 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 90e421216c854ac3a9ae7559f8554d0a 2024-11-27T01:25:49,207 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/.tmp/c/90e421216c854ac3a9ae7559f8554d0a as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/c/90e421216c854ac3a9ae7559f8554d0a 2024-11-27T01:25:49,214 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 90e421216c854ac3a9ae7559f8554d0a 2024-11-27T01:25:49,214 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/c/90e421216c854ac3a9ae7559f8554d0a, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-27T01:25:49,214 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 2bcce029151ca4a5343818abe3b79c28 in 32ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-27T01:25:49,215 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/0000000000000003002 2024-11-27T01:25:49,216 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:49,217 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:49,217 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T01:25:49,219 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2bcce029151ca4a5343818abe3b79c28 2024-11-27T01:25:49,222 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenIntoWAL/2bcce029151ca4a5343818abe3b79c28/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-27T01:25:49,223 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2bcce029151ca4a5343818abe3b79c28; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63087630, jitterRate=-0.059921056032180786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T01:25:49,223 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2bcce029151ca4a5343818abe3b79c28: Writing region info on filesystem at 1732670748425Initializing all the Stores at 1732670748426 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670748426Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670748426Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670748426Cleaning up temporary data from old regions at 1732670749217 (+791 ms)Region opened successfully at 1732670749223 (+6 ms) 2024-11-27T01:25:49,288 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2bcce029151ca4a5343818abe3b79c28, disabling compactions & flushes 2024-11-27T01:25:49,288 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:49,288 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:49,288 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. after waiting 0 ms 2024-11-27T01:25:49,288 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:49,291 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732670743864.2bcce029151ca4a5343818abe3b79c28. 2024-11-27T01:25:49,291 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2bcce029151ca4a5343818abe3b79c28: Waiting for close lock at 1732670749288Disabling compacts and flushes for region at 1732670749288Disabling writes for close at 1732670749288Writing region close event to WAL at 1732670749291 (+3 ms)Closed at 1732670749291 2024-11-27T01:25:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741857_1034 (size=175) 2024-11-27T01:25:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741857_1034 (size=175) 2024-11-27T01:25:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741857_1034 (size=175) 2024-11-27T01:25:49,299 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:49,299 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670748377) 2024-11-27T01:25:49,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=402 (was 386) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:36155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45477 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:45933 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1003118666_22 at /127.0.0.1:48362 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1003118666_22 at /127.0.0.1:47618 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:36155 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1003118666_22 at /127.0.0.1:57718 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45933 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:45477 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1003118666_22 at /127.0.0.1:47594 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 275), ProcessCount=11 (was 11), AvailableMemoryMB=6092 (was 6234) 2024-11-27T01:25:49,328 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=6091 2024-11-27T01:25:49,350 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:49,353 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:49,354 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:49,357 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30587466, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-30587466, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:49,371 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30587466/hregion-30587466.1732670749358, exclude list is [], retry=0 2024-11-27T01:25:49,377 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:49,378 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:49,386 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T01:25:49,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:49,388 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:49,389 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:25:49,389 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T01:25:49,389 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T01:25:49,389 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:49,391 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-27T01:25:49,391 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:49,421 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30587466/hregion-30587466.1732670749358 2024-11-27T01:25:49,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:49,423 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3fdad5a00e8e8af9ac1c44d9b0129370, NAME => 'test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:25:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741865_1042 (size=43) 2024-11-27T01:25:49,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741865_1042 (size=43) 2024-11-27T01:25:49,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741865_1042 (size=43) 2024-11-27T01:25:49,444 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:49,446 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,447 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fdad5a00e8e8af9ac1c44d9b0129370 columnFamilyName a 2024-11-27T01:25:49,447 DEBUG [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:49,448 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(327): Store=3fdad5a00e8e8af9ac1c44d9b0129370/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:49,448 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,450 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fdad5a00e8e8af9ac1c44d9b0129370 columnFamilyName b 2024-11-27T01:25:49,451 DEBUG [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:49,451 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(327): Store=3fdad5a00e8e8af9ac1c44d9b0129370/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:49,451 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,453 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fdad5a00e8e8af9ac1c44d9b0129370 columnFamilyName c 2024-11-27T01:25:49,453 DEBUG [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:49,454 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(327): Store=3fdad5a00e8e8af9ac1c44d9b0129370/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:49,454 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,458 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,458 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,458 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:49,460 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:49,464 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:49,465 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3fdad5a00e8e8af9ac1c44d9b0129370; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73836967, jitterRate=0.10025654733181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:49,466 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3fdad5a00e8e8af9ac1c44d9b0129370: Writing region info on filesystem at 1732670749444Initializing all the Stores at 1732670749445 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670749445Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670749445Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670749445Cleaning up temporary data from old regions at 1732670749458 (+13 ms)Region opened successfully at 1732670749466 (+8 ms) 2024-11-27T01:25:49,467 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3fdad5a00e8e8af9ac1c44d9b0129370, disabling compactions & flushes 2024-11-27T01:25:49,467 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:49,467 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:49,467 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. after waiting 0 ms 2024-11-27T01:25:49,467 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:49,468 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:49,468 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3fdad5a00e8e8af9ac1c44d9b0129370: Waiting for close lock at 1732670749467Disabling compacts and flushes for region at 1732670749467Disabling writes for close at 1732670749467Writing region close event to WAL at 1732670749467Closed at 1732670749467 2024-11-27T01:25:49,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741864_1041 (size=175) 2024-11-27T01:25:49,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741864_1041 (size=175) 2024-11-27T01:25:49,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741864_1041 (size=175) 2024-11-27T01:25:49,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:49,475 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30587466:(num 1732670749358) 2024-11-27T01:25:49,475 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:49,479 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:49,501 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480, exclude list is [], retry=0 2024-11-27T01:25:49,505 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:49,505 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:49,505 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:49,536 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 2024-11-27T01:25:49,537 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:49,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741866_1043 (size=371388) 2024-11-27T01:25:49,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741866_1043 (size=371388) 2024-11-27T01:25:49,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741866_1043 (size=371388) 2024-11-27T01:25:49,705 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480, size=362.7 K (371388bytes) 2024-11-27T01:25:49,705 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 2024-11-27T01:25:49,706 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 after 1ms 2024-11-27T01:25:49,738 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:49,739 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 took 34ms 2024-11-27T01:25:49,744 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732670749480.temp 2024-11-27T01:25:49,776 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000000001-wal.1732670749480.temp 2024-11-27T01:25:49,791 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 so closing down 2024-11-27T01:25:49,791 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:49,793 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741867_1044 (size=371388) 2024-11-27T01:25:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741867_1044 (size=371388) 2024-11-27T01:25:49,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741867_1044 (size=371388) 2024-11-27T01:25:49,801 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000000001-wal.1732670749480.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-11-27T01:25:49,803 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000000001-wal.1732670749480.temp to hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003000 2024-11-27T01:25:49,803 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 63 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-27T01:25:49,803 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480, journal: Splitting hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480, size=362.7 K (371388bytes) at 1732670749705Split 1024 edits, skipped 0 edits. at 1732670749759 (+54 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000000001-wal.1732670749480.temp at 1732670749776 (+17 ms)Split 2048 edits, skipped 0 edits. at 1732670749777 (+1 ms)Finishing writing output for hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 so closing down at 1732670749791 (+14 ms)3 split writer threads finished at 1732670749793 (+2 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000000001-wal.1732670749480.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1732670749801 (+8 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000000001-wal.1732670749480.temp to hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003000 at 1732670749803 (+2 ms)Processed 3000 edits across 1 Regions in 63 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732670749803 2024-11-27T01:25:49,805 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749480 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670749480 2024-11-27T01:25:49,806 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003000 2024-11-27T01:25:49,806 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:49,808 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:49,823 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809, exclude list is [], retry=0 2024-11-27T01:25:49,826 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:49,826 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:49,827 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:49,856 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 2024-11-27T01:25:49,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:50,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741868_1045 (size=371515) 2024-11-27T01:25:50,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741868_1045 (size=371515) 2024-11-27T01:25:50,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741868_1045 (size=371515) 2024-11-27T01:25:50,026 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809, size=362.8 K (371515bytes) 2024-11-27T01:25:50,026 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 2024-11-27T01:25:50,027 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 after 0ms 2024-11-27T01:25:50,057 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:50,058 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 took 32ms 2024-11-27T01:25:50,063 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732670749809.temp 2024-11-27T01:25:50,128 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 so closing down 2024-11-27T01:25:50,128 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:50,146 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003001-wal.1732670749809.temp 2024-11-27T01:25:50,160 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:50,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741869_1046 (size=371515) 2024-11-27T01:25:50,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741869_1046 (size=371515) 2024-11-27T01:25:50,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741869_1046 (size=371515) 2024-11-27T01:25:50,166 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003001-wal.1732670749809.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-11-27T01:25:50,169 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003001-wal.1732670749809.temp to hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000006000 2024-11-27T01:25:50,169 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-27T01:25:50,169 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809, journal: Splitting hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809, size=362.8 K (371515bytes) at 1732670750026Split 1024 edits, skipped 0 edits. at 1732670750100 (+74 ms)Split 2048 edits, skipped 0 edits. at 1732670750107 (+7 ms)Finishing writing output for hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 so closing down at 1732670750128 (+21 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003001-wal.1732670749809.temp at 1732670750146 (+18 ms)3 split writer threads finished at 1732670750160 (+14 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003001-wal.1732670749809.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1732670750166 (+6 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003001-wal.1732670749809.temp to hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000006000 at 1732670750169 (+3 ms)Processed 3000 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732670750169 2024-11-27T01:25:50,171 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670749809 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670749809 2024-11-27T01:25:50,172 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000006000 2024-11-27T01:25:50,172 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:50,174 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/test2727-manual,16010,1732670749349, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:50,188 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670750175, exclude list is [], retry=0 2024-11-27T01:25:50,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:50,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:50,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:50,222 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732670749349/wal.1732670750175 2024-11-27T01:25:50,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:50,222 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fdad5a00e8e8af9ac1c44d9b0129370, NAME => 'test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:50,223 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:50,223 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,223 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,225 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,226 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fdad5a00e8e8af9ac1c44d9b0129370 columnFamilyName a 2024-11-27T01:25:50,226 DEBUG [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,227 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(327): Store=3fdad5a00e8e8af9ac1c44d9b0129370/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,227 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,228 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fdad5a00e8e8af9ac1c44d9b0129370 columnFamilyName b 2024-11-27T01:25:50,228 DEBUG [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,229 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(327): Store=3fdad5a00e8e8af9ac1c44d9b0129370/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,229 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,230 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fdad5a00e8e8af9ac1c44d9b0129370 columnFamilyName c 2024-11-27T01:25:50,230 DEBUG [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,231 INFO [StoreOpener-3fdad5a00e8e8af9ac1c44d9b0129370-1 {}] regionserver.HStore(327): Store=3fdad5a00e8e8af9ac1c44d9b0129370/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,231 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,232 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,234 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,235 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003000 2024-11-27T01:25:50,265 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:50,289 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003000 2024-11-27T01:25:50,290 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000006000 2024-11-27T01:25:50,319 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:50,341 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000006000 2024-11-27T01:25:50,341 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3fdad5a00e8e8af9ac1c44d9b0129370 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-27T01:25:50,362 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/a/fe0e3a1ad72241af80724a3d3df1d033 is 41, key is test2727/a:100/1732670749863/Put/seqid=0 2024-11-27T01:25:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741871_1048 (size=84227) 2024-11-27T01:25:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741871_1048 (size=84227) 2024-11-27T01:25:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741871_1048 (size=84227) 2024-11-27T01:25:50,373 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/a/fe0e3a1ad72241af80724a3d3df1d033 2024-11-27T01:25:50,399 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/b/000c7b3eb7af473ca9840f08d2bb0131 is 41, key is test2727/b:100/1732670749917/Put/seqid=0 2024-11-27T01:25:50,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741872_1049 (size=84609) 2024-11-27T01:25:50,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741872_1049 (size=84609) 2024-11-27T01:25:50,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741872_1049 (size=84609) 2024-11-27T01:25:50,405 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/b/000c7b3eb7af473ca9840f08d2bb0131 2024-11-27T01:25:50,441 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/c/2903251d23f14cb3b514a87d4eb5b1c2 is 41, key is test2727/c:100/1732670749962/Put/seqid=0 2024-11-27T01:25:50,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741873_1050 (size=84609) 2024-11-27T01:25:50,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741873_1050 (size=84609) 2024-11-27T01:25:50,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741873_1050 (size=84609) 2024-11-27T01:25:50,449 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/c/2903251d23f14cb3b514a87d4eb5b1c2 2024-11-27T01:25:50,457 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/a/fe0e3a1ad72241af80724a3d3df1d033 as hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/a/fe0e3a1ad72241af80724a3d3df1d033 2024-11-27T01:25:50,463 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/a/fe0e3a1ad72241af80724a3d3df1d033, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-27T01:25:50,464 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/b/000c7b3eb7af473ca9840f08d2bb0131 as hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/b/000c7b3eb7af473ca9840f08d2bb0131 2024-11-27T01:25:50,470 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/b/000c7b3eb7af473ca9840f08d2bb0131, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T01:25:50,471 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/.tmp/c/2903251d23f14cb3b514a87d4eb5b1c2 as hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/c/2903251d23f14cb3b514a87d4eb5b1c2 2024-11-27T01:25:50,477 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/c/2903251d23f14cb3b514a87d4eb5b1c2, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-27T01:25:50,477 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 3fdad5a00e8e8af9ac1c44d9b0129370 in 136ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-27T01:25:50,477 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000003000 2024-11-27T01:25:50,478 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/0000000000000006000 2024-11-27T01:25:50,479 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,479 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,480 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:50,481 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3fdad5a00e8e8af9ac1c44d9b0129370 2024-11-27T01:25:50,484 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/test2727/3fdad5a00e8e8af9ac1c44d9b0129370/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-27T01:25:50,485 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3fdad5a00e8e8af9ac1c44d9b0129370; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70263941, jitterRate=0.04701431095600128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:50,486 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3fdad5a00e8e8af9ac1c44d9b0129370: Writing region info on filesystem at 1732670750223Initializing all the Stores at 1732670750224 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750224Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750224Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750224Obtaining lock to block concurrent updates at 1732670750341 (+117 ms)Preparing flush snapshotting stores in 3fdad5a00e8e8af9ac1c44d9b0129370 at 1732670750341Finished memstore snapshotting test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732670750341Flushing stores of test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. at 1732670750341Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/a: creating writer at 1732670750342 (+1 ms)Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/a: appending metadata at 1732670750361 (+19 ms)Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/a: closing flushed file at 1732670750361Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/b: creating writer at 1732670750379 (+18 ms)Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/b: appending metadata at 1732670750397 (+18 ms)Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/b: closing flushed file at 1732670750397Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/c: creating writer at 1732670750411 (+14 ms)Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/c: appending metadata at 1732670750440 (+29 ms)Flushing 3fdad5a00e8e8af9ac1c44d9b0129370/c: closing flushed file at 1732670750440Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10a7a55f: reopening flushed file at 1732670750456 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3682ed6d: reopening flushed file at 1732670750463 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b24ca60: reopening flushed file at 1732670750470 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 3fdad5a00e8e8af9ac1c44d9b0129370 in 136ms, sequenceid=6000, compaction requested=false; wal=null at 1732670750477 (+7 ms)Cleaning up temporary data from old regions at 1732670750479 (+2 ms)Region opened successfully at 1732670750486 (+7 ms) 2024-11-27T01:25:50,487 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-27T01:25:50,487 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3fdad5a00e8e8af9ac1c44d9b0129370, disabling compactions & flushes 2024-11-27T01:25:50,487 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:50,488 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:50,488 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. after waiting 0 ms 2024-11-27T01:25:50,488 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:50,489 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732670749351.3fdad5a00e8e8af9ac1c44d9b0129370. 2024-11-27T01:25:50,489 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3fdad5a00e8e8af9ac1c44d9b0129370: Waiting for close lock at 1732670750487Disabling compacts and flushes for region at 1732670750487Disabling writes for close at 1732670750488 (+1 ms)Writing region close event to WAL at 1732670750489 (+1 ms)Closed at 1732670750489 2024-11-27T01:25:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741870_1047 (size=175) 2024-11-27T01:25:50,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741870_1047 (size=175) 2024-11-27T01:25:50,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741870_1047 (size=175) 2024-11-27T01:25:50,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:50,496 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670750175) 2024-11-27T01:25:50,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404 (was 402) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:48428 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57830 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:47720 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=257 (was 253) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5889 (was 6091) 2024-11-27T01:25:50,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=11, AvailableMemoryMB=5888 2024-11-27T01:25:50,538 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:50,545 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:50,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732670750545 2024-11-27T01:25:50,578 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545 2024-11-27T01:25:50,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:25:50,581 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 378b7d7c7045d246f335c6e242205f12, NAME => 'testSequentialEditLogSeqNum,,1732670750539.378b7d7c7045d246f335c6e242205f12.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:50,581 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732670750539.378b7d7c7045d246f335c6e242205f12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:50,581 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,581 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,582 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12 doesn't exist for region: 378b7d7c7045d246f335c6e242205f12 on table testSequentialEditLogSeqNum 2024-11-27T01:25:50,582 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 378b7d7c7045d246f335c6e242205f12 on table testSequentialEditLogSeqNum 2024-11-27T01:25:50,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741875_1052 (size=62) 2024-11-27T01:25:50,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741875_1052 (size=62) 2024-11-27T01:25:50,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741875_1052 (size=62) 2024-11-27T01:25:50,592 INFO [StoreOpener-378b7d7c7045d246f335c6e242205f12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,594 INFO [StoreOpener-378b7d7c7045d246f335c6e242205f12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 378b7d7c7045d246f335c6e242205f12 columnFamilyName a 2024-11-27T01:25:50,594 DEBUG [StoreOpener-378b7d7c7045d246f335c6e242205f12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,594 INFO [StoreOpener-378b7d7c7045d246f335c6e242205f12-1 {}] regionserver.HStore(327): Store=378b7d7c7045d246f335c6e242205f12/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,595 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,595 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,596 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,596 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,596 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,598 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 378b7d7c7045d246f335c6e242205f12 2024-11-27T01:25:50,601 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:50,601 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 378b7d7c7045d246f335c6e242205f12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61285041, jitterRate=-0.08678172528743744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:25:50,602 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 378b7d7c7045d246f335c6e242205f12: Writing region info on filesystem at 1732670750581Initializing all the Stores at 1732670750592 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750592Cleaning up temporary data from old regions at 1732670750596 (+4 ms)Region opened successfully at 1732670750602 (+6 ms) 2024-11-27T01:25:50,616 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 378b7d7c7045d246f335c6e242205f12 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-27T01:25:50,641 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/.tmp/a/4463fdb74dd54e57bd2802b84a9038ec is 81, key is testSequentialEditLogSeqNum/a:x0/1732670750602/Put/seqid=0 2024-11-27T01:25:50,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741876_1053 (size=5833) 2024-11-27T01:25:50,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741876_1053 (size=5833) 2024-11-27T01:25:50,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741876_1053 (size=5833) 2024-11-27T01:25:50,649 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/.tmp/a/4463fdb74dd54e57bd2802b84a9038ec 2024-11-27T01:25:50,657 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/.tmp/a/4463fdb74dd54e57bd2802b84a9038ec as hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/a/4463fdb74dd54e57bd2802b84a9038ec 2024-11-27T01:25:50,663 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/a/4463fdb74dd54e57bd2802b84a9038ec, entries=10, sequenceid=13, filesize=5.7 K 2024-11-27T01:25:50,665 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 378b7d7c7045d246f335c6e242205f12 in 49ms, sequenceid=13, compaction requested=false 2024-11-27T01:25:50,665 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 378b7d7c7045d246f335c6e242205f12: 2024-11-27T01:25:50,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:25:50,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:25:50,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:25:50,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:25:50,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-27T01:25:50,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741874_1051 (size=3256) 2024-11-27T01:25:50,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741874_1051 (size=3256) 2024-11-27T01:25:50,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741874_1051 (size=3256) 2024-11-27T01:25:50,699 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545, size=3.2 K (3256bytes) 2024-11-27T01:25:50,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545 2024-11-27T01:25:50,700 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545 after 1ms 2024-11-27T01:25:50,736 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:50,736 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545 took 37ms 2024-11-27T01:25:50,740 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545 so closing down 2024-11-27T01:25:50,740 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:50,744 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670750545.temp 2024-11-27T01:25:50,777 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000003-wal.1732670750545.temp 2024-11-27T01:25:50,778 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:50,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741877_1054 (size=2650) 2024-11-27T01:25:50,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741877_1054 (size=2650) 2024-11-27T01:25:50,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741877_1054 (size=2650) 2024-11-27T01:25:50,795 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000003-wal.1732670750545.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-27T01:25:50,796 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000003-wal.1732670750545.temp to hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000020 2024-11-27T01:25:50,798 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 61 ms; skipped=2; WAL=hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-27T01:25:50,798 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545, journal: Splitting hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545, size=3.2 K (3256bytes) at 1732670750699Finishing writing output for hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545 so closing down at 1732670750740 (+41 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000003-wal.1732670750545.temp at 1732670750777 (+37 ms)3 split writer threads finished at 1732670750778 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000003-wal.1732670750545.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732670750795 (+17 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000003-wal.1732670750545.temp to hdfs://localhost:36155/hbase/data/default/testSequentialEditLogSeqNum/378b7d7c7045d246f335c6e242205f12/recovered.edits/0000000000000000020 at 1732670750796 (+1 ms)Processed 17 edits across 1 Regions in 61 ms; skipped=2; WAL=hdfs://localhost:36155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732670750538/wal.1732670750545, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732670750798 (+2 ms) 2024-11-27T01:25:50,814 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:48428 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57830 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:47720 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=257 (was 257), ProcessCount=11 (was 11), AvailableMemoryMB=5894 (was 5888) - AvailableMemoryMB LEAK? - 2024-11-27T01:25:50,830 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=257, ProcessCount=11, AvailableMemoryMB=5894 2024-11-27T01:25:50,850 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:50,854 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:50,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:50,896 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23503602, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-23503602, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:50,910 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23503602/hregion-23503602.1732670750897, exclude list is [], retry=0 2024-11-27T01:25:50,914 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:50,914 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:50,914 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:50,947 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23503602/hregion-23503602.1732670750897 2024-11-27T01:25:50,948 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:50,948 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2be42e47f6dac71a28fad6265a19ac8f, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:25:50,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741879_1056 (size=70) 2024-11-27T01:25:50,959 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:50,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741879_1056 (size=70) 2024-11-27T01:25:50,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741879_1056 (size=70) 2024-11-27T01:25:50,960 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,962 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName a 2024-11-27T01:25:50,962 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,963 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,963 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,964 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName b 2024-11-27T01:25:50,965 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,965 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,965 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,967 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName c 2024-11-27T01:25:50,967 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:50,967 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:50,968 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,968 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,970 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,970 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,971 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:50,972 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:50,974 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:50,975 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2be42e47f6dac71a28fad6265a19ac8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67779211, jitterRate=0.009988948702812195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:50,975 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2be42e47f6dac71a28fad6265a19ac8f: Writing region info on filesystem at 1732670750959Initializing all the Stores at 1732670750960 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750960Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750960Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670750960Cleaning up temporary data from old regions at 1732670750970 (+10 ms)Region opened successfully at 1732670750975 (+5 ms) 2024-11-27T01:25:50,975 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2be42e47f6dac71a28fad6265a19ac8f, disabling compactions & flushes 2024-11-27T01:25:50,975 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:50,975 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:50,975 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. after waiting 0 ms 2024-11-27T01:25:50,976 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:50,976 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:50,976 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2be42e47f6dac71a28fad6265a19ac8f: Waiting for close lock at 1732670750975Disabling compacts and flushes for region at 1732670750975Disabling writes for close at 1732670750975Writing region close event to WAL at 1732670750976 (+1 ms)Closed at 1732670750976 2024-11-27T01:25:50,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741878_1055 (size=175) 2024-11-27T01:25:50,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741878_1055 (size=175) 2024-11-27T01:25:50,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741878_1055 (size=175) 2024-11-27T01:25:50,985 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:50,985 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23503602:(num 1732670750897) 2024-11-27T01:25:50,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:50,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:51,000 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987, exclude list is [], retry=0 2024-11-27T01:25:51,003 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:51,004 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:51,004 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:51,033 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 2024-11-27T01:25:51,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2be42e47f6dac71a28fad6265a19ac8f, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,036 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,037 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName a 2024-11-27T01:25:51,037 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:51,038 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:51,038 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,039 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName b 2024-11-27T01:25:51,039 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:51,040 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:51,040 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,041 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName c 2024-11-27T01:25:51,041 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:51,041 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:51,042 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,042 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,043 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,045 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,045 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,045 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:51,047 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:51,048 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2be42e47f6dac71a28fad6265a19ac8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72484681, jitterRate=0.08010591566562653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:51,048 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2be42e47f6dac71a28fad6265a19ac8f: Writing region info on filesystem at 1732670751034Initializing all the Stores at 1732670751035 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670751035Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670751036 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670751036Cleaning up temporary data from old regions at 1732670751045 (+9 ms)Region opened successfully at 1732670751048 (+3 ms) 2024-11-27T01:25:51,052 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732670751052/Put/seqid=0 2024-11-27T01:25:51,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741881_1058 (size=4826) 2024-11-27T01:25:51,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741881_1058 (size=4826) 2024-11-27T01:25:51,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741881_1058 (size=4826) 2024-11-27T01:25:51,061 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 2be42e47f6dac71a28fad6265a19ac8f/a 2024-11-27T01:25:51,069 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-27T01:25:51,069 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:25:51,069 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2be42e47f6dac71a28fad6265a19ac8f: 2024-11-27T01:25:51,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/e69df2ee1ab747098f0866485dacd61a_SeqId_3_ 2024-11-27T01:25:51,072 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2be42e47f6dac71a28fad6265a19ac8f/a as hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/e69df2ee1ab747098f0866485dacd61a_SeqId_3_ - updating store file list. 2024-11-27T01:25:51,077 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e69df2ee1ab747098f0866485dacd61a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T01:25:51,077 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/e69df2ee1ab747098f0866485dacd61a_SeqId_3_ into 2be42e47f6dac71a28fad6265a19ac8f/a 2024-11-27T01:25:51,077 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2be42e47f6dac71a28fad6265a19ac8f/a (new location: hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/e69df2ee1ab747098f0866485dacd61a_SeqId_3_) 2024-11-27T01:25:51,119 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987, size=0 (0bytes) 2024-11-27T01:25:51,119 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 might be still open, length is 0 2024-11-27T01:25:51,119 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 2024-11-27T01:25:51,120 WARN [IPC Server handler 2 on default port 36155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-27T01:25:51,120 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 after 1ms 2024-11-27T01:25:52,956 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:37182 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37182 dst: /127.0.0.1:39261 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39261 remote=/127.0.0.1:37182]. Total timeout mills is 60000, 58127 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:52,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:40338 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45665:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40338 dst: /127.0.0.1:45665 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:52,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57014 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57014 dst: /127.0.0.1:44005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:52,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741880_1059 (size=606) 2024-11-27T01:25:52,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741880_1059 (size=606) 2024-11-27T01:25:55,121 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 after 4002ms 2024-11-27T01:25:55,157 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:55,157 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 took 4038ms 2024-11-27T01:25:55,159 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987; continuing. 2024-11-27T01:25:55,159 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 so closing down 2024-11-27T01:25:55,159 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:55,161 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732670750987.temp 2024-11-27T01:25:55,191 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005-wal.1732670750987.temp 2024-11-27T01:25:55,191 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:55,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741882_1060 (size=329) 2024-11-27T01:25:55,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741882_1060 (size=329) 2024-11-27T01:25:55,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741882_1060 (size=329) 2024-11-27T01:25:55,202 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005-wal.1732670750987.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-27T01:25:55,204 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005-wal.1732670750987.temp to hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005 2024-11-27T01:25:55,204 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 47 ms; skipped=1; WAL=hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:25:55,205 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987, journal: Splitting hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987, size=0 (0bytes) at 1732670751119Finishing writing output for hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 so closing down at 1732670755159 (+4040 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005-wal.1732670750987.temp at 1732670755191 (+32 ms)3 split writer threads finished at 1732670755191Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005-wal.1732670750987.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732670755202 (+11 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005-wal.1732670750987.temp to hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005 at 1732670755204 (+2 ms)Processed 2 edits across 1 Regions in 47 ms; skipped=1; WAL=hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987, size=0, length=0, corrupted=false, cancelled=false at 1732670755204 2024-11-27T01:25:55,207 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670750987 2024-11-27T01:25:55,208 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005 2024-11-27T01:25:55,208 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:55,211 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:55,233 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670755211, exclude list is [], retry=0 2024-11-27T01:25:55,237 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:55,238 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:55,238 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:55,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670755211 2024-11-27T01:25:55,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:55,292 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2be42e47f6dac71a28fad6265a19ac8f, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:55,292 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:55,292 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,292 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,295 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,296 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName a 2024-11-27T01:25:55,296 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:55,307 DEBUG [StoreFileOpener-2be42e47f6dac71a28fad6265a19ac8f-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e69df2ee1ab747098f0866485dacd61a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-27T01:25:55,307 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/e69df2ee1ab747098f0866485dacd61a_SeqId_3_ 2024-11-27T01:25:55,307 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:55,307 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,309 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName b 2024-11-27T01:25:55,309 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:55,310 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:55,310 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,311 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2be42e47f6dac71a28fad6265a19ac8f columnFamilyName c 2024-11-27T01:25:55,311 DEBUG [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:55,312 INFO [StoreOpener-2be42e47f6dac71a28fad6265a19ac8f-1 {}] regionserver.HStore(327): Store=2be42e47f6dac71a28fad6265a19ac8f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:55,312 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,316 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005 2024-11-27T01:25:55,362 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:55,364 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005 2024-11-27T01:25:55,364 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2be42e47f6dac71a28fad6265a19ac8f 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-27T01:25:55,391 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/.tmp/a/ecbc473c454e405782689ca34e3fd86e is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732670751083/Put/seqid=0 2024-11-27T01:25:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741884_1062 (size=5149) 2024-11-27T01:25:55,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741884_1062 (size=5149) 2024-11-27T01:25:55,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741884_1062 (size=5149) 2024-11-27T01:25:55,418 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/.tmp/a/ecbc473c454e405782689ca34e3fd86e 2024-11-27T01:25:55,431 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/.tmp/a/ecbc473c454e405782689ca34e3fd86e as hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/ecbc473c454e405782689ca34e3fd86e 2024-11-27T01:25:55,440 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/a/ecbc473c454e405782689ca34e3fd86e, entries=1, sequenceid=5, filesize=5.0 K 2024-11-27T01:25:55,440 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2be42e47f6dac71a28fad6265a19ac8f in 76ms, sequenceid=5, compaction requested=false; wal=null 2024-11-27T01:25:55,441 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/0000000000000000005 2024-11-27T01:25:55,443 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,443 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,444 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:55,446 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2be42e47f6dac71a28fad6265a19ac8f 2024-11-27T01:25:55,450 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2be42e47f6dac71a28fad6265a19ac8f/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T01:25:55,453 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2be42e47f6dac71a28fad6265a19ac8f; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66452891, jitterRate=-0.009774759411811829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:55,453 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2be42e47f6dac71a28fad6265a19ac8f: Writing region info on filesystem at 1732670755293Initializing all the Stores at 1732670755294 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670755294Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670755294Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670755294Obtaining lock to block concurrent updates at 1732670755364 (+70 ms)Preparing flush snapshotting stores in 2be42e47f6dac71a28fad6265a19ac8f at 1732670755364Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732670755364Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. at 1732670755364Flushing 2be42e47f6dac71a28fad6265a19ac8f/a: creating writer at 1732670755364Flushing 2be42e47f6dac71a28fad6265a19ac8f/a: appending metadata at 1732670755390 (+26 ms)Flushing 2be42e47f6dac71a28fad6265a19ac8f/a: closing flushed file at 1732670755390Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@458ee12f: reopening flushed file at 1732670755425 (+35 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2be42e47f6dac71a28fad6265a19ac8f in 76ms, sequenceid=5, compaction requested=false; wal=null at 1732670755440 (+15 ms)Cleaning up temporary data from old regions at 1732670755443 (+3 ms)Region opened successfully at 1732670755453 (+10 ms) 2024-11-27T01:25:55,459 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2be42e47f6dac71a28fad6265a19ac8f, disabling compactions & flushes 2024-11-27T01:25:55,459 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:55,459 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:55,459 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. after waiting 0 ms 2024-11-27T01:25:55,459 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:55,461 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732670750851.2be42e47f6dac71a28fad6265a19ac8f. 2024-11-27T01:25:55,461 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2be42e47f6dac71a28fad6265a19ac8f: Waiting for close lock at 1732670755459Disabling compacts and flushes for region at 1732670755459Disabling writes for close at 1732670755459Writing region close event to WAL at 1732670755460 (+1 ms)Closed at 1732670755461 (+1 ms) 2024-11-27T01:25:55,469 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670755211 not finished, retry = 0 2024-11-27T01:25:55,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741883_1061 (size=175) 2024-11-27T01:25:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741883_1061 (size=175) 2024-11-27T01:25:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741883_1061 (size=175) 2024-11-27T01:25:55,572 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:55,572 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670755211) 2024-11-27T01:25:55,587 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-357019725_22 at /127.0.0.1:37198 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:36155 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-357019725_22 at /127.0.0.1:57044 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:36155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-357019725_22 at /127.0.0.1:40360 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 257) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6497 (was 5894) - AvailableMemoryMB LEAK? - 2024-11-27T01:25:55,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=6495 2024-11-27T01:25:55,616 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:55,623 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T01:25:55,627 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:55,631 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ebe9604 2024-11-27T01:25:55,632 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T01:25:55,634 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T01:25:55,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T01:25:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T01:25:55,648 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T01:25:55,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-27T01:25:55,651 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:55,653 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T01:25:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:25:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741885_1063 (size=694) 2024-11-27T01:25:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741885_1063 (size=694) 2024-11-27T01:25:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741885_1063 (size=694) 2024-11-27T01:25:55,678 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1 2024-11-27T01:25:55,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741886_1064 (size=77) 2024-11-27T01:25:55,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741886_1064 (size=77) 2024-11-27T01:25:55,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741886_1064 (size=77) 2024-11-27T01:25:55,690 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:55,690 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:25:55,690 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:55,690 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:55,690 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:25:55,690 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:55,690 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:55,690 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670755690Disabling compacts and flushes for region at 1732670755690Disabling writes for close at 1732670755690Writing region close event to WAL at 1732670755690Closed at 1732670755690 2024-11-27T01:25:55,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T01:25:55,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732670755693"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732670755693"}]},"ts":"1732670755693"} 2024-11-27T01:25:55,702 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T01:25:55,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T01:25:55,707 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732670755703"}]},"ts":"1732670755703"} 2024-11-27T01:25:55,712 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-27T01:25:55,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-27T01:25:55,714 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:25:55,714 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:25:55,714 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-27T01:25:55,714 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:25:55,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN}] 2024-11-27T01:25:55,718 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN 2024-11-27T01:25:55,720 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN; state=OFFLINE, location=7d6f26a4c136,41943,1732670739675; forceNewPlan=false, retain=false 2024-11-27T01:25:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:25:55,873 INFO [7d6f26a4c136:37429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:25:55,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPENING, regionLocation=7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:55,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN because future has completed 2024-11-27T01:25:55,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675}] 2024-11-27T01:25:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:25:56,034 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:25:56,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46927, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:25:56,042 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,042 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:56,043 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,043 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:56,043 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,043 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,045 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,046 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf1 2024-11-27T01:25:56,046 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:56,047 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:56,047 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,048 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf2 2024-11-27T01:25:56,048 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:56,049 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:56,049 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,050 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,050 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,051 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,051 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,052 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:25:56,054 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,057 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:56,057 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6ed6a1e752c9fdb4bc0ccc2dad778184; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65665947, jitterRate=-0.02150113880634308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:25:56,057 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,058 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Running coprocessor pre-open hook at 1732670756043Writing region info on filesystem at 1732670756043Initializing all the Stores at 1732670756044 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670756044Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670756045 (+1 ms)Cleaning up temporary data from old regions at 1732670756051 (+6 ms)Running coprocessor post-open hooks at 1732670756057 (+6 ms)Region opened successfully at 1732670756058 (+1 ms) 2024-11-27T01:25:56,060 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., pid=6, masterSystemTime=1732670756034 2024-11-27T01:25:56,063 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,064 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPEN, openSeqNum=2, regionLocation=7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:56,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675 because future has completed 2024-11-27T01:25:56,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T01:25:56,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675 in 189 msec 2024-11-27T01:25:56,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T01:25:56,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN in 357 msec 2024-11-27T01:25:56,077 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T01:25:56,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732670756077"}]},"ts":"1732670756077"} 2024-11-27T01:25:56,080 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-27T01:25:56,082 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T01:25:56,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 442 msec 2024-11-27T01:25:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T01:25:56,296 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-27T01:25:56,296 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-27T01:25:56,298 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:25:56,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-27T01:25:56,303 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T01:25:56,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-27T01:25:56,315 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=2] 2024-11-27T01:25:56,317 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T01:25:56,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T01:25:56,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6ed6a1e752c9fdb4bc0ccc2dad778184, source=7d6f26a4c136,41943,1732670739675, destination=7d6f26a4c136,39245,1732670739775, warming up region on 7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:56,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6ed6a1e752c9fdb4bc0ccc2dad778184, source=7d6f26a4c136,41943,1732670739675, destination=7d6f26a4c136,39245,1732670739775, running balancer 2024-11-27T01:25:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE 2024-11-27T01:25:56,337 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE 2024-11-27T01:25:56,340 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=CLOSING, regionLocation=7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:56,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:56,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:56,341 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE because future has completed 2024-11-27T01:25:56,343 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf1 2024-11-27T01:25:56,343 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:56,343 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:25:56,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675}] 2024-11-27T01:25:56,345 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:56,345 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,346 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf2 2024-11-27T01:25:56,346 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:56,347 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:25:56,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:25:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670756347Disabling compacts and flushes for region at 1732670756347Disabling writes for close at 1732670756347Writing region close event to WAL at 1732670756355 (+8 ms)Closed at 1732670756355 2024-11-27T01:25:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-27T01:25:56,503 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,504 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:25:56,504 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:25:56,504 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,504 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,504 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:25:56,504 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,505 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 6ed6a1e752c9fdb4bc0ccc2dad778184 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-27T01:25:56,522 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/af2a328196714afbb4af65843be22b03 is 35, key is r1/cf1:q/1732670756319/Put/seqid=0 2024-11-27T01:25:56,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741887_1065 (size=4783) 2024-11-27T01:25:56,531 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/af2a328196714afbb4af65843be22b03 2024-11-27T01:25:56,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741887_1065 (size=4783) 2024-11-27T01:25:56,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741887_1065 (size=4783) 2024-11-27T01:25:56,539 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/af2a328196714afbb4af65843be22b03 as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03 2024-11-27T01:25:56,546 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03, entries=1, sequenceid=5, filesize=4.7 K 2024-11-27T01:25:56,547 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6ed6a1e752c9fdb4bc0ccc2dad778184 in 42ms, sequenceid=5, compaction requested=false 2024-11-27T01:25:56,547 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-27T01:25:56,552 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T01:25:56,555 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,555 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670756504Running coprocessor pre-close hooks at 1732670756504Disabling compacts and flushes for region at 1732670756504Disabling writes for close at 1732670756504Obtaining lock to block concurrent updates at 1732670756505 (+1 ms)Preparing flush snapshotting stores in 6ed6a1e752c9fdb4bc0ccc2dad778184 at 1732670756505Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732670756505Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. at 1732670756506 (+1 ms)Flushing 6ed6a1e752c9fdb4bc0ccc2dad778184/cf1: creating writer at 1732670756506Flushing 6ed6a1e752c9fdb4bc0ccc2dad778184/cf1: appending metadata at 1732670756522 (+16 ms)Flushing 6ed6a1e752c9fdb4bc0ccc2dad778184/cf1: closing flushed file at 1732670756522Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a0b1f: reopening flushed file at 1732670756538 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6ed6a1e752c9fdb4bc0ccc2dad778184 in 42ms, sequenceid=5, compaction requested=false at 1732670756547 (+9 ms)Writing region close event to WAL at 1732670756549 (+2 ms)Running coprocessor post-close hooks at 1732670756553 (+4 ms)Closed at 1732670756555 (+2 ms) 2024-11-27T01:25:56,556 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 6ed6a1e752c9fdb4bc0ccc2dad778184 move to 7d6f26a4c136,39245,1732670739775 record at close sequenceid=5 2024-11-27T01:25:56,559 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,560 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=CLOSED 2024-11-27T01:25:56,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675 because future has completed 2024-11-27T01:25:56,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T01:25:56,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675 in 222 msec 2024-11-27T01:25:56,570 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,39245,1732670739775; forceNewPlan=false, retain=false 2024-11-27T01:25:56,720 INFO [7d6f26a4c136:37429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:25:56,721 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPENING, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:56,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE because future has completed 2024-11-27T01:25:56,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775}] 2024-11-27T01:25:56,885 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,885 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:56,886 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,886 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:56,886 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,886 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,891 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,893 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf1 2024-11-27T01:25:56,893 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:56,900 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03 2024-11-27T01:25:56,900 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:56,900 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,901 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf2 2024-11-27T01:25:56,901 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:56,902 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:56,902 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,903 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,904 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,905 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,905 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,906 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:25:56,907 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,908 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6ed6a1e752c9fdb4bc0ccc2dad778184; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69389459, jitterRate=0.033983513712882996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:25:56,908 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:56,909 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Running coprocessor pre-open hook at 1732670756886Writing region info on filesystem at 1732670756886Initializing all the Stores at 1732670756888 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670756888Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670756891 (+3 ms)Cleaning up temporary data from old regions at 1732670756905 (+14 ms)Running coprocessor post-open hooks at 1732670756908 (+3 ms)Region opened successfully at 1732670756909 (+1 ms) 2024-11-27T01:25:56,910 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., pid=9, masterSystemTime=1732670756880 2024-11-27T01:25:56,913 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,913 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:56,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPEN, openSeqNum=9, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:56,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775 because future has completed 2024-11-27T01:25:56,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-27T01:25:56,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775 in 193 msec 2024-11-27T01:25:56,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE in 587 msec 2024-11-27T01:25:56,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:25:56,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:25:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:58612 deadline: 1732670816952, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=39245 startCode=1732670739775. As of locationSeqNum=5. 2024-11-27T01:25:56,962 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=39245 startCode=1732670739775. As of locationSeqNum=5. 2024-11-27T01:25:56,963 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=39245 startCode=1732670739775. As of locationSeqNum=5. 2024-11-27T01:25:56,963 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,39245,1732670739775, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=39245 startCode=1732670739775. As of locationSeqNum=5. 2024-11-27T01:25:57,087 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6ed6a1e752c9fdb4bc0ccc2dad778184 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-27T01:25:57,111 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/42b771192a3a476e8afd3dd680850c27 is 29, key is r1/cf1:/1732670757076/DeleteFamily/seqid=0 2024-11-27T01:25:57,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741888_1066 (size=4906) 2024-11-27T01:25:57,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741888_1066 (size=4906) 2024-11-27T01:25:57,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741888_1066 (size=4906) 2024-11-27T01:25:57,129 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,139 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,160 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf2/38fd0889ab764a66bdca2a8fc3e8046a is 29, key is r1/cf2:/1732670757076/DeleteFamily/seqid=0 2024-11-27T01:25:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741889_1067 (size=4906) 2024-11-27T01:25:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741889_1067 (size=4906) 2024-11-27T01:25:57,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741889_1067 (size=4906) 2024-11-27T01:25:57,176 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf2/38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,184 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,185 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/42b771192a3a476e8afd3dd680850c27 as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,192 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,192 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T01:25:57,193 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf2/38fd0889ab764a66bdca2a8fc3e8046a as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,201 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,201 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a, entries=1, sequenceid=12, filesize=4.8 K 2024-11-27T01:25:57,203 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 6ed6a1e752c9fdb4bc0ccc2dad778184 in 115ms, sequenceid=12, compaction requested=false 2024-11-27T01:25:57,203 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: 2024-11-27T01:25:57,206 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T01:25:57,208 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6ed6a1e752c9fdb4bc0ccc2dad778184/cf1 is initiating major compaction (all files) 2024-11-27T01:25:57,208 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:25:57,208 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:57,208 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6ed6a1e752c9fdb4bc0ccc2dad778184/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,209 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03, hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27] into tmpdir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp, totalSize=9.5 K 2024-11-27T01:25:57,210 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting af2a328196714afbb4af65843be22b03, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732670756319 2024-11-27T01:25:57,210 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 42b771192a3a476e8afd3dd680850c27, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T01:25:57,222 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6ed6a1e752c9fdb4bc0ccc2dad778184#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:25:57,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741890_1068 (size=4626) 2024-11-27T01:25:57,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741890_1068 (size=4626) 2024-11-27T01:25:57,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741890_1068 (size=4626) 2024-11-27T01:25:57,245 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf1/b2018546aa7c4ce086f723c2b3807757 as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/b2018546aa7c4ce086f723c2b3807757 2024-11-27T01:25:57,259 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 6ed6a1e752c9fdb4bc0ccc2dad778184/cf1 of 6ed6a1e752c9fdb4bc0ccc2dad778184 into b2018546aa7c4ce086f723c2b3807757(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:25:57,260 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: 2024-11-27T01:25:57,260 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-27T01:25:57,260 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6ed6a1e752c9fdb4bc0ccc2dad778184/cf2 is initiating major compaction (all files) 2024-11-27T01:25:57,260 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T01:25:57,260 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T01:25:57,260 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6ed6a1e752c9fdb4bc0ccc2dad778184/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,261 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a] into tmpdir=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp, totalSize=4.8 K 2024-11-27T01:25:57,261 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 38fd0889ab764a66bdca2a8fc3e8046a, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-27T01:25:57,268 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6ed6a1e752c9fdb4bc0ccc2dad778184#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T01:25:57,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741891_1069 (size=4592) 2024-11-27T01:25:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741891_1069 (size=4592) 2024-11-27T01:25:57,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741891_1069 (size=4592) 2024-11-27T01:25:57,297 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/.tmp/cf2/adc38d5081e04bd29329b47e2ae79b4a as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/adc38d5081e04bd29329b47e2ae79b4a 2024-11-27T01:25:57,305 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 6ed6a1e752c9fdb4bc0ccc2dad778184/cf2 of 6ed6a1e752c9fdb4bc0ccc2dad778184 into adc38d5081e04bd29329b47e2ae79b4a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:25:57,305 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: 2024-11-27T01:25:57,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6ed6a1e752c9fdb4bc0ccc2dad778184, source=7d6f26a4c136,39245,1732670739775, destination=7d6f26a4c136,41943,1732670739675, warming up region on 7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:57,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6ed6a1e752c9fdb4bc0ccc2dad778184, source=7d6f26a4c136,39245,1732670739775, destination=7d6f26a4c136,41943,1732670739675, running balancer 2024-11-27T01:25:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE 2024-11-27T01:25:57,316 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE 2024-11-27T01:25:57,318 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=CLOSING, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:57,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:57,319 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,320 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf1 2024-11-27T01:25:57,320 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:57,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE because future has completed 2024-11-27T01:25:57,322 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T01:25:57,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775}] 2024-11-27T01:25:57,332 INFO [StoreFileOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,332 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,339 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03 2024-11-27T01:25:57,345 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/b2018546aa7c4ce086f723c2b3807757 2024-11-27T01:25:57,345 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:57,345 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,346 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf2 2024-11-27T01:25:57,347 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:57,354 INFO [StoreFileOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,354 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,360 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/adc38d5081e04bd29329b47e2ae79b4a 2024-11-27T01:25:57,360 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:25:57,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41943 {}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670757360Disabling compacts and flushes for region at 1732670757360Disabling writes for close at 1732670757360Writing region close event to WAL at 1732670757363 (+3 ms)Closed at 1732670757363 2024-11-27T01:25:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-27T01:25:57,477 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,478 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T01:25:57,478 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:25:57,478 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,478 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,478 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:25:57,478 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,478 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03, hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27] to archive 2024-11-27T01:25:57,482 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:25:57,487 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03 to hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/af2a328196714afbb4af65843be22b03 2024-11-27T01:25:57,491 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27 to hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/42b771192a3a476e8afd3dd680850c27 2024-11-27T01:25:57,507 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a] to archive 2024-11-27T01:25:57,509 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:25:57,511 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a to hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/38fd0889ab764a66bdca2a8fc3e8046a 2024-11-27T01:25:57,520 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-27T01:25:57,521 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,521 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670757478Running coprocessor pre-close hooks at 1732670757478Disabling compacts and flushes for region at 1732670757478Disabling writes for close at 1732670757478Writing region close event to WAL at 1732670757513 (+35 ms)Running coprocessor post-close hooks at 1732670757521 (+8 ms)Closed at 1732670757521 2024-11-27T01:25:57,521 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 6ed6a1e752c9fdb4bc0ccc2dad778184 move to 7d6f26a4c136,41943,1732670739675 record at close sequenceid=12 2024-11-27T01:25:57,524 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,525 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=CLOSED 2024-11-27T01:25:57,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775 because future has completed 2024-11-27T01:25:57,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T01:25:57,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775 in 209 msec 2024-11-27T01:25:57,537 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE; state=CLOSED, location=7d6f26a4c136,41943,1732670739675; forceNewPlan=false, retain=false 2024-11-27T01:25:57,687 INFO [7d6f26a4c136:37429 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T01:25:57,688 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPENING, regionLocation=7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:57,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE because future has completed 2024-11-27T01:25:57,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675}] 2024-11-27T01:25:57,848 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,849 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:57,849 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,849 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:57,849 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,849 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,851 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,852 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf1 2024-11-27T01:25:57,852 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:57,859 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/b2018546aa7c4ce086f723c2b3807757 2024-11-27T01:25:57,859 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:57,859 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,860 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf2 2024-11-27T01:25:57,860 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:57,866 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/adc38d5081e04bd29329b47e2ae79b4a 2024-11-27T01:25:57,867 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:57,867 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,868 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,869 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,870 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,870 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,871 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:25:57,873 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,874 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 6ed6a1e752c9fdb4bc0ccc2dad778184; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72720723, jitterRate=0.08362321555614471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:25:57,874 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,874 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Running coprocessor pre-open hook at 1732670757849Writing region info on filesystem at 1732670757849Initializing all the Stores at 1732670757850 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670757850Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670757850Cleaning up temporary data from old regions at 1732670757870 (+20 ms)Running coprocessor post-open hooks at 1732670757874 (+4 ms)Region opened successfully at 1732670757874 2024-11-27T01:25:57,876 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., pid=12, masterSystemTime=1732670757843 2024-11-27T01:25:57,879 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,880 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,881 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPEN, openSeqNum=18, regionLocation=7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:57,882 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37429 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7d6f26a4c136,41943,1732670739675, table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T01:25:57,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675 because future has completed 2024-11-27T01:25:57,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-27T01:25:57,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,41943,1732670739675 in 198 msec 2024-11-27T01:25:57,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, REOPEN/MOVE in 578 msec 2024-11-27T01:25:57,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:25:57,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:25:57,922 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 7d6f26a4c136,41943,1732670739675: testing ***** 2024-11-27T01:25:57,922 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T01:25:57,924 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-27T01:25:57,925 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-27T01:25:57,930 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-27T01:25:57,932 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-27T01:25:57,950 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 481795736 }, "NonHeapMemoryUsage": { "committed": 172556288, "init": 7667712, "max": -1, "used": 169892960 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 49, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 6, "ProcessCallTime_95th_percentile": 19, "ProcessCallTime_98th_percentile": 49, "ProcessCallTime_99th_percentile": 49, "ProcessCallTime_99.9th_percentile": 49, "ProcessCallTime_TimeRangeCount_0-1": 10, "ProcessCallTime_TimeRangeCount_1-3": 14, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 49, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 20, "TotalCallTime_98th_percentile": 49, "TotalCallTime_99th_percentile": 49, "TotalCallTime_99.9th_percentile": 49, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 14, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5782 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58346", "tag.serverName": "7d6f26a4c136,39245,1732670739775", "tag.clusterId": "09523c17-5740-4230-8a95-38d6b0bf24fa", "tag.Context": "regionserver", "tag.Hostname": "7d6f26a4c136", "regionCount": 1, "storeCount": 4, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 952, "memStoreHeapSize": 2584, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732670739775, "averageRegionSize": 952, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.6, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 532480, "ByteBuffAllocatorHeapAllocationRatio": 0.400849396798649, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 28, "totalRowActionRequestCount": 12, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 5, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 16, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 45, "MajorCompactionTime_max": 49, "MajorCompactionTime_mean": 47, "MajorCompactionTime_25th_percentile": 45, "MajorCompactionTime_median": 46, "MajorCompactionTime_75th_percentile": 49, "MajorCompactionTime_90th_percentile": 49, "MajorCompactionTime_95th_percentile": 49, "MajorCompactionTime_98th_percentile": 49, "MajorCompactionTime_99th_percentile": 49, "MajorCompactionTime_99.9th_percentile": 49, "MajorCompactionTime_TimeRangeCount_30-100": 2, "ScanTime_num_ops": 9, "ScanTime_min": 1, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 15, "Put_min": 1, "Put_max": 3, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_0-1": 4, "Put_TimeRangeCount_1-3": 9, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 3, "Delete_max": 3, "Delete_mean": 3, "Delete_25th_percentile": 3, "Delete_median": 3, "Delete_75th_percentile": 3, "Delete_90th_percentile": 3, "Delete_95th_percentile": 3, "Delete_98th_percentile": 3, "Delete_99th_percentile": 3, "Delete_99.9th_percentile": 3, "Delete_TimeRangeCount_1-3": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 45, "CompactionTime_max": 49, "CompactionTime_mean": 47, "CompactionTime_25th_percentile": 45, "CompactionTime_median": 46, "CompactionTime_75th_percentile": 49, "CompactionTime_90th_percentile": 49, "CompactionTime_95th_percentile": 49, "CompactionTime_98th_percentile": 49, "CompactionTime_99th_percentile": 49, "CompactionTime_99.9th_percentile": 49, "CompactionTime_TimeRangeCount_30-100": 2, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 115, "FlushTime_max": 115, "FlushTime_mean": 115, "FlushTime_25th_percentile": 115, "FlushTime_median": 115, "FlushTime_75th_percentile": 115, "FlushTime_90th_percentile": 115, "FlushTime_95th_percentile": 115, "FlushTime_98th_percentile": 115, "FlushTime_99th_percentile": 115, "FlushTime_99.9th_percentile": 115, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 1112, "ScanSize_mean": 889, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_0-10": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-27T01:25:57,954 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37429 {}] master.MasterRpcServices(700): 7d6f26a4c136,41943,1732670739675 reported a fatal error: ***** ABORTING region server 7d6f26a4c136,41943,1732670739675: testing ***** 2024-11-27T01:25:57,957 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,41943,1732670739675' ***** 2024-11-27T01:25:57,957 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-27T01:25:57,957 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:25:57,957 INFO [RS:0;7d6f26a4c136:41943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-27T01:25:57,958 INFO [RS:0;7d6f26a4c136:41943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-27T01:25:57,958 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,958 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(956): aborting server 7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:57,958 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:25:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39245 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:33132 deadline: 1732670817958, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=41943 startCode=1732670739675. As of locationSeqNum=12. 2024-11-27T01:25:57,959 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:25:57,959 INFO [RS:0;7d6f26a4c136:41943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7d6f26a4c136:41943. 2024-11-27T01:25:57,959 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,959 DEBUG [RS:0;7d6f26a4c136:41943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:25:57,959 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,959 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:25:57,959 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,959 DEBUG [RS:0;7d6f26a4c136:41943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:25:57,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,39245,1732670739775, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,39245,1732670739775, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=41943 startCode=1732670739675. As of locationSeqNum=12. 2024-11-27T01:25:57,960 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T01:25:57,960 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,39245,1732670739775, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=41943 startCode=1732670739675. As of locationSeqNum=12. 2024-11-27T01:25:57,960 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1325): Online Regions={6ed6a1e752c9fdb4bc0ccc2dad778184=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.} 2024-11-27T01:25:57,960 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,39245,1732670739775, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=7d6f26a4c136 port=41943 startCode=1732670739675. As of locationSeqNum=12. 2024-11-27T01:25:57,960 DEBUG [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1351): Waiting on 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:57,960 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:25:57,963 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:57,963 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670757958Running coprocessor pre-close hooks at 1732670757958Disabling compacts and flushes for region at 1732670757959 (+1 ms)Disabling writes for close at 1732670757959Writing region close event to WAL at 1732670757961 (+2 ms)Running coprocessor post-close hooks at 1732670757962 (+1 ms)Closed at 1732670757963 (+1 ms) 2024-11-27T01:25:57,963 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:58,011 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:25:58,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7d6f26a4c136,41943,1732670739675 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:58,067 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7d6f26a4c136,41943,1732670739675 aborting 2024-11-27T01:25:58,067 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 7d6f26a4c136,41943,1732670739675 aborting 2024-11-27T01:25:58,067 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=12 from cache 2024-11-27T01:25:58,085 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T01:25:58,086 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T01:25:58,160 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,41943,1732670739675; all regions closed. 2024-11-27T01:25:58,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741835_1011 (size=2288) 2024-11-27T01:25:58,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741835_1011 (size=2288) 2024-11-27T01:25:58,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741835_1011 (size=2288) 2024-11-27T01:25:58,166 DEBUG [RS:0;7d6f26a4c136:41943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:25:58,166 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:25:58,166 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:25:58,166 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:25:58,167 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:25:58,167 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:25:58,167 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:25:58,167 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:25:58,167 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:25:58,167 INFO [RS:0;7d6f26a4c136:41943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41943 2024-11-27T01:25:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,41943,1732670739675 2024-11-27T01:25:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:25:58,176 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:25:58,177 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,41943,1732670739675] 2024-11-27T01:25:58,179 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,41943,1732670739675 already deleted, retry=false 2024-11-27T01:25:58,179 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7d6f26a4c136,41943,1732670739675 on 7d6f26a4c136,37429,1732670738986 2024-11-27T01:25:58,184 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7d6f26a4c136,41943,1732670739675, splitWal=true, meta=false 2024-11-27T01:25:58,186 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 7d6f26a4c136,41943,1732670739675 (carryingMeta=false) 7d6f26a4c136,41943,1732670739675/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@d1d9b39[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T01:25:58,187 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7d6f26a4c136,41943,1732670739675, splitWal=true, meta=false 2024-11-27T01:25:58,189 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 7d6f26a4c136,41943,1732670739675 had 1 regions 2024-11-27T01:25:58,190 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7d6f26a4c136,41943,1732670739675, splitWal=true, meta=false, isMeta: false 2024-11-27T01:25:58,192 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting 2024-11-27T01:25:58,193 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7d6f26a4c136,41943,1732670739675 WAL count=1, meta=false 2024-11-27T01:25:58,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C41943%2C1732670739675.1732670741264}] 2024-11-27T01:25:58,200 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:58,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C41943%2C1732670739675.1732670741264, worker=7d6f26a4c136,43859,1732670739814}] 2024-11-27T01:25:58,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:25:58,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x101570739b00001, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:25:58,279 INFO [RS:0;7d6f26a4c136:41943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:25:58,279 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18] 2024-11-27T01:25:58,280 INFO [RS:0;7d6f26a4c136:41943 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,41943,1732670739675; zookeeper connection closed. 2024-11-27T01:25:58,280 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e7be129 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e7be129 2024-11-27T01:25:58,281 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7d6f26a4c136:41943 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7d6f26a4c136/172.17.0.2:41943 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:25:58,282 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18, error=java.net.ConnectException: Call to address=7d6f26a4c136:41943 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7d6f26a4c136/172.17.0.2:41943 2024-11-27T01:25:58,282 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-27T01:25:58,282 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18 from cache 2024-11-27T01:25:58,282 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 7d6f26a4c136:41943 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 7d6f26a4c136/172.17.0.2:41943 2024-11-27T01:25:58,362 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T01:25:58,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59251, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T01:25:58,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43859 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-27T01:25:58,392 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264, size=2.2 K (2288bytes) 2024-11-27T01:25:58,392 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 2024-11-27T01:25:58,393 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 after 1ms 2024-11-27T01:25:58,443 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:58,443 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 took 51ms 2024-11-27T01:25:58,459 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 6ed6a1e752c9fdb4bc0ccc2dad778184: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-27T01:25:58,459 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 so closing down 2024-11-27T01:25:58,459 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:58,460 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:25:58,460 INFO [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 12 ms; skipped=6; WAL=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-27T01:25:58,460 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264, journal: Splitting hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264, size=2.2 K (2288bytes) at 1732670758392Finishing writing output for hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 so closing down at 1732670758459 (+67 ms)3 split writer threads finished at 1732670758460 (+1 ms)Processed 6 edits across 0 Regions in 12 ms; skipped=6; WAL=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732670758460 2024-11-27T01:25:58,460 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 2024-11-27T01:25:58,462 DEBUG [RS_LOG_REPLAY_OPS-regionserver/7d6f26a4c136:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-27T01:25:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37429 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-27T01:25:58,472 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting/7d6f26a4c136%2C41943%2C1732670739675.1732670741264 to hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs 2024-11-27T01:25:58,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T01:25:58,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 7d6f26a4c136%2C41943%2C1732670739675.1732670741264, worker=7d6f26a4c136,43859,1732670739814 in 270 msec 2024-11-27T01:25:58,478 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=7d6f26a4c136,43859,1732670739814 2024-11-27T01:25:58,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T01:25:58,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 7d6f26a4c136%2C41943%2C1732670739675.1732670741264, worker=7d6f26a4c136,43859,1732670739814 in 285 msec 2024-11-27T01:25:58,484 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting dir is empty, no logs to split. 2024-11-27T01:25:58,484 INFO [PEWorker-1 {}] master.SplitWALManager(105): 7d6f26a4c136,41943,1732670739675 WAL count=0, meta=false 2024-11-27T01:25:58,484 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 7d6f26a4c136,41943,1732670739675 WAL splitting is done? wals=0, meta=false 2024-11-27T01:25:58,487 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 7d6f26a4c136,41943,1732670739675 failed, ignore...File hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/WALs/7d6f26a4c136,41943,1732670739675-splitting does not exist. 2024-11-27T01:25:58,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN}] 2024-11-27T01:25:58,492 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN 2024-11-27T01:25:58,493 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T01:25:58,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18] 2024-11-27T01:25:58,589 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 7d6f26a4c136:41943 this server is in the failed servers list 2024-11-27T01:25:58,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=7d6f26a4c136:41943 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7d6f26a4c136:41943 2024-11-27T01:25:58,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 7d6f26a4c136:41943 2024-11-27T01:25:58,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,41943,1732670739675, seqNum=18 from cache 2024-11-27T01:25:58,644 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(204): Hosts are {7d6f26a4c136=0} racks are {/default-rack=0} 2024-11-27T01:25:58,644 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T01:25:58,644 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T01:25:58,644 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T01:25:58,644 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T01:25:58,644 INFO [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T01:25:58,644 INFO [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T01:25:58,645 DEBUG [7d6f26a4c136:37429 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T01:25:58,645 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPENING, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:58,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN because future has completed 2024-11-27T01:25:58,649 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775}] 2024-11-27T01:25:58,806 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:58,806 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ed6a1e752c9fdb4bc0ccc2dad778184, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:58,807 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,807 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:58,807 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,807 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,809 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,809 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf1 2024-11-27T01:25:58,810 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:58,816 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf1/b2018546aa7c4ce086f723c2b3807757 2024-11-27T01:25:58,816 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:58,817 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,817 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed6a1e752c9fdb4bc0ccc2dad778184 columnFamilyName cf2 2024-11-27T01:25:58,818 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:58,824 DEBUG [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/cf2/adc38d5081e04bd29329b47e2ae79b4a 2024-11-27T01:25:58,824 INFO [StoreOpener-6ed6a1e752c9fdb4bc0ccc2dad778184-1 {}] regionserver.HStore(327): Store=6ed6a1e752c9fdb4bc0ccc2dad778184/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:58,824 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,825 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,826 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,827 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,827 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,827 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T01:25:58,829 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,829 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 6ed6a1e752c9fdb4bc0ccc2dad778184; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68133390, jitterRate=0.015266627073287964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T01:25:58,830 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:25:58,830 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Running coprocessor pre-open hook at 1732670758807Writing region info on filesystem at 1732670758807Initializing all the Stores at 1732670758808 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670758808Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670758808Cleaning up temporary data from old regions at 1732670758827 (+19 ms)Running coprocessor post-open hooks at 1732670758830 (+3 ms)Region opened successfully at 1732670758830 2024-11-27T01:25:58,831 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., pid=17, masterSystemTime=1732670758802 2024-11-27T01:25:58,833 DEBUG [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:58,834 INFO [RS_OPEN_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:25:58,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=6ed6a1e752c9fdb4bc0ccc2dad778184, regionState=OPEN, openSeqNum=18, regionLocation=7d6f26a4c136,39245,1732670739775 2024-11-27T01:25:58,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775 because future has completed 2024-11-27T01:25:58,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T01:25:58,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ed6a1e752c9fdb4bc0ccc2dad778184, server=7d6f26a4c136,39245,1732670739775 in 189 msec 2024-11-27T01:25:58,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-27T01:25:58,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6ed6a1e752c9fdb4bc0ccc2dad778184, ASSIGN in 350 msec 2024-11-27T01:25:58,843 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 7d6f26a4c136,41943,1732670739675 after splitting done 2024-11-27T01:25:58,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 7d6f26a4c136,41943,1732670739675, splitWal=true, meta=false in 663 msec 2024-11-27T01:25:59,098 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184., hostname=7d6f26a4c136,39245,1732670739775, seqNum=18] 2024-11-27T01:25:59,113 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 411), OpenFileDescriptor=1049 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=6284 (was 6495) 2024-11-27T01:25:59,115 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-11-27T01:25:59,127 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1049, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=6284 2024-11-27T01:25:59,127 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1049 is superior to 1024 2024-11-27T01:25:59,143 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:59,145 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:25:59,146 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:25:59,148 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19916357, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-19916357, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:59,161 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19916357/hregion-19916357.1732670759149, exclude list is [], retry=0 2024-11-27T01:25:59,165 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:59,165 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:59,165 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:59,197 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19916357/hregion-19916357.1732670759149 2024-11-27T01:25:59,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:25:59,198 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a64150cbe93b472de6b265e6406bd4c1, NAME => 'testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:25:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741893_1071 (size=67) 2024-11-27T01:25:59,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741893_1071 (size=67) 2024-11-27T01:25:59,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741893_1071 (size=67) 2024-11-27T01:25:59,211 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:59,213 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,217 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName a 2024-11-27T01:25:59,217 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:59,218 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:59,218 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,219 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName b 2024-11-27T01:25:59,219 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:59,220 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:59,220 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,221 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName c 2024-11-27T01:25:59,221 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:59,222 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:59,222 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,223 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,225 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,225 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,225 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:59,226 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,229 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:25:59,230 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a64150cbe93b472de6b265e6406bd4c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74955312, jitterRate=0.11692118644714355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:59,231 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a64150cbe93b472de6b265e6406bd4c1: Writing region info on filesystem at 1732670759212Initializing all the Stores at 1732670759213 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670759213Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670759213Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670759213Cleaning up temporary data from old regions at 1732670759225 (+12 ms)Region opened successfully at 1732670759231 (+6 ms) 2024-11-27T01:25:59,231 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a64150cbe93b472de6b265e6406bd4c1, disabling compactions & flushes 2024-11-27T01:25:59,231 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,231 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,231 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. after waiting 0 ms 2024-11-27T01:25:59,231 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,232 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,232 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a64150cbe93b472de6b265e6406bd4c1: Waiting for close lock at 1732670759231Disabling compacts and flushes for region at 1732670759231Disabling writes for close at 1732670759231Writing region close event to WAL at 1732670759231Closed at 1732670759231 2024-11-27T01:25:59,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741892_1070 (size=175) 2024-11-27T01:25:59,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741892_1070 (size=175) 2024-11-27T01:25:59,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741892_1070 (size=175) 2024-11-27T01:25:59,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:25:59,238 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19916357:(num 1732670759149) 2024-11-27T01:25:59,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:25:59,240 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:25:59,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241, exclude list is [], retry=0 2024-11-27T01:25:59,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:25:59,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:25:59,264 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:25:59,298 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 2024-11-27T01:25:59,299 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:25:59,299 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a64150cbe93b472de6b265e6406bd4c1, NAME => 'testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:25:59,299 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:25:59,299 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,299 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,301 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,302 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName a 2024-11-27T01:25:59,302 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:59,302 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:59,302 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,303 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName b 2024-11-27T01:25:59,303 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:59,304 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:59,304 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,304 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName c 2024-11-27T01:25:59,305 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:25:59,305 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:25:59,305 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,307 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,309 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,309 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,309 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:25:59,311 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:25:59,311 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a64150cbe93b472de6b265e6406bd4c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62045610, jitterRate=-0.07544836401939392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:25:59,312 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a64150cbe93b472de6b265e6406bd4c1: Writing region info on filesystem at 1732670759299Initializing all the Stores at 1732670759300 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670759300Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670759300Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670759300Cleaning up temporary data from old regions at 1732670759309 (+9 ms)Region opened successfully at 1732670759312 (+3 ms) 2024-11-27T01:25:59,354 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a64150cbe93b472de6b265e6406bd4c1 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T01:25:59,374 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/a/45f3a8df1a29418e99888509a7236b30 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732670759312/Put/seqid=0 2024-11-27T01:25:59,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741895_1073 (size=5958) 2024-11-27T01:25:59,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-27T01:25:59,386 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:59,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741895_1073 (size=5958) 2024-11-27T01:25:59,389 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/a/45f3a8df1a29418e99888509a7236b30 2024-11-27T01:25:59,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741895_1073 (size=5958) 2024-11-27T01:25:59,394 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-27T01:25:59,394 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:59,394 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-27T01:25:59,394 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:59,395 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-27T01:25:59,395 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-27T01:25:59,417 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/b/3ac5974bf3894acca0092f9ed66b9fe7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732670759322/Put/seqid=0 2024-11-27T01:25:59,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741896_1074 (size=5958) 2024-11-27T01:25:59,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741896_1074 (size=5958) 2024-11-27T01:25:59,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741896_1074 (size=5958) 2024-11-27T01:25:59,441 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/b/3ac5974bf3894acca0092f9ed66b9fe7 2024-11-27T01:25:59,466 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/c/2d0a0b534ccb44a9b17bddcccaafdc48 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732670759342/Put/seqid=0 2024-11-27T01:25:59,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741897_1075 (size=5958) 2024-11-27T01:25:59,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741897_1075 (size=5958) 2024-11-27T01:25:59,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741897_1075 (size=5958) 2024-11-27T01:25:59,893 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/c/2d0a0b534ccb44a9b17bddcccaafdc48 2024-11-27T01:25:59,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/a/45f3a8df1a29418e99888509a7236b30 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/a/45f3a8df1a29418e99888509a7236b30 2024-11-27T01:25:59,904 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/a/45f3a8df1a29418e99888509a7236b30, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T01:25:59,905 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/b/3ac5974bf3894acca0092f9ed66b9fe7 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/b/3ac5974bf3894acca0092f9ed66b9fe7 2024-11-27T01:25:59,910 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/b/3ac5974bf3894acca0092f9ed66b9fe7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T01:25:59,911 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/c/2d0a0b534ccb44a9b17bddcccaafdc48 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/c/2d0a0b534ccb44a9b17bddcccaafdc48 2024-11-27T01:25:59,916 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/c/2d0a0b534ccb44a9b17bddcccaafdc48, entries=10, sequenceid=33, filesize=5.8 K 2024-11-27T01:25:59,917 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a64150cbe93b472de6b265e6406bd4c1 in 563ms, sequenceid=33, compaction requested=false 2024-11-27T01:25:59,917 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a64150cbe93b472de6b265e6406bd4c1: 2024-11-27T01:25:59,917 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a64150cbe93b472de6b265e6406bd4c1, disabling compactions & flushes 2024-11-27T01:25:59,917 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,918 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,918 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. after waiting 0 ms 2024-11-27T01:25:59,918 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,919 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. 2024-11-27T01:25:59,919 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a64150cbe93b472de6b265e6406bd4c1: Waiting for close lock at 1732670759917Disabling compacts and flushes for region at 1732670759917Disabling writes for close at 1732670759918 (+1 ms)Writing region close event to WAL at 1732670759919 (+1 ms)Closed at 1732670759919 2024-11-27T01:25:59,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741894_1072 (size=6311) 2024-11-27T01:25:59,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741894_1072 (size=6311) 2024-11-27T01:25:59,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741894_1072 (size=6311) 2024-11-27T01:25:59,928 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/b/3ac5974bf3894acca0092f9ed66b9fe7 to hdfs://localhost:36155/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/b/3ac5974bf3894acca0092f9ed66b9fe7 2024-11-27T01:25:59,947 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241, size=6.2 K (6311bytes) 2024-11-27T01:25:59,947 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 2024-11-27T01:25:59,947 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 after 0ms 2024-11-27T01:25:59,976 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:25:59,976 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 took 30ms 2024-11-27T01:25:59,979 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 so closing down 2024-11-27T01:25:59,979 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:25:59,980 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670759241.temp 2024-11-27T01:26:00,006 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000003-wal.1732670759241.temp 2024-11-27T01:26:00,006 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:26:00,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741898_1076 (size=5575) 2024-11-27T01:26:00,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741898_1076 (size=5575) 2024-11-27T01:26:00,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741898_1076 (size=5575) 2024-11-27T01:26:00,013 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000003-wal.1732670759241.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T01:26:00,015 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000003-wal.1732670759241.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000032 2024-11-27T01:26:00,015 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-27T01:26:00,015 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241, journal: Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241, size=6.2 K (6311bytes) at 1732670759947Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 so closing down at 1732670759979 (+32 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000003-wal.1732670759241.temp at 1732670760006 (+27 ms)3 split writer threads finished at 1732670760006Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000003-wal.1732670759241.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732670760013 (+7 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000003-wal.1732670759241.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000032 at 1732670760015 (+2 ms)Processed 32 edits across 1 Regions in 38 ms; skipped=2; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732670760015 2024-11-27T01:26:00,016 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670759241 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670759241 2024-11-27T01:26:00,017 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000032 2024-11-27T01:26:00,017 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:00,019 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:00,032 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670760019, exclude list is [], retry=0 2024-11-27T01:26:00,034 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:00,035 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:00,035 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:00,062 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670760019 2024-11-27T01:26:00,063 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:26:00,063 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a64150cbe93b472de6b265e6406bd4c1, NAME => 'testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:00,063 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:00,063 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,063 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,065 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,065 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName a 2024-11-27T01:26:00,066 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,071 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/a/45f3a8df1a29418e99888509a7236b30 2024-11-27T01:26:00,071 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,071 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,072 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName b 2024-11-27T01:26:00,072 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,073 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,073 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,073 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a64150cbe93b472de6b265e6406bd4c1 columnFamilyName c 2024-11-27T01:26:00,074 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,078 DEBUG [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/c/2d0a0b534ccb44a9b17bddcccaafdc48 2024-11-27T01:26:00,078 INFO [StoreOpener-a64150cbe93b472de6b265e6406bd4c1-1 {}] regionserver.HStore(327): Store=a64150cbe93b472de6b265e6406bd4c1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,078 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,079 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,080 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,081 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000032 2024-11-27T01:26:00,109 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:00,111 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000032 2024-11-27T01:26:00,111 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a64150cbe93b472de6b265e6406bd4c1 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T01:26:00,129 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/b/3785e0d5f9db480d96dbbcb596fcbeaf is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732670759322/Put/seqid=0 2024-11-27T01:26:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741900_1078 (size=5958) 2024-11-27T01:26:00,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741900_1078 (size=5958) 2024-11-27T01:26:00,138 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/b/3785e0d5f9db480d96dbbcb596fcbeaf 2024-11-27T01:26:00,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741900_1078 (size=5958) 2024-11-27T01:26:00,145 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/.tmp/b/3785e0d5f9db480d96dbbcb596fcbeaf as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/b/3785e0d5f9db480d96dbbcb596fcbeaf 2024-11-27T01:26:00,151 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/b/3785e0d5f9db480d96dbbcb596fcbeaf, entries=10, sequenceid=32, filesize=5.8 K 2024-11-27T01:26:00,151 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a64150cbe93b472de6b265e6406bd4c1 in 40ms, sequenceid=32, compaction requested=false; wal=null 2024-11-27T01:26:00,152 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/0000000000000000032 2024-11-27T01:26:00,153 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,154 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,154 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:00,156 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a64150cbe93b472de6b265e6406bd4c1 2024-11-27T01:26:00,158 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/a64150cbe93b472de6b265e6406bd4c1/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-27T01:26:00,160 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a64150cbe93b472de6b265e6406bd4c1; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62846435, jitterRate=-0.06351514160633087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:00,160 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a64150cbe93b472de6b265e6406bd4c1: Writing region info on filesystem at 1732670760063Initializing all the Stores at 1732670760064 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760064Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760064Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760064Obtaining lock to block concurrent updates at 1732670760111 (+47 ms)Preparing flush snapshotting stores in a64150cbe93b472de6b265e6406bd4c1 at 1732670760111Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732670760111Flushing stores of testReplayEditsWrittenViaHRegion,,1732670759144.a64150cbe93b472de6b265e6406bd4c1. at 1732670760111Flushing a64150cbe93b472de6b265e6406bd4c1/b: creating writer at 1732670760112 (+1 ms)Flushing a64150cbe93b472de6b265e6406bd4c1/b: appending metadata at 1732670760128 (+16 ms)Flushing a64150cbe93b472de6b265e6406bd4c1/b: closing flushed file at 1732670760128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33324de5: reopening flushed file at 1732670760144 (+16 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a64150cbe93b472de6b265e6406bd4c1 in 40ms, sequenceid=32, compaction requested=false; wal=null at 1732670760152 (+8 ms)Cleaning up temporary data from old regions at 1732670760154 (+2 ms)Region opened successfully at 1732670760160 (+6 ms) 2024-11-27T01:26:00,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:37198 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57178 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57044 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:40544 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:37312 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:40360 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1123 (was 1049) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=6201 (was 6284) 2024-11-27T01:26:00,183 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1123 is superior to 1024 2024-11-27T01:26:00,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1123, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=6200 2024-11-27T01:26:00,196 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1123 is superior to 1024 2024-11-27T01:26:00,212 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:00,214 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:00,214 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:26:00,217 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08745447, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-08745447, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:00,230 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08745447/hregion-08745447.1732670760217, exclude list is [], retry=0 2024-11-27T01:26:00,232 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:00,232 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:00,233 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:00,260 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08745447/hregion-08745447.1732670760217 2024-11-27T01:26:00,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:26:00,260 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bf7b4102551d01886a513742a3c1c7d9, NAME => 'testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:26:00,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741902_1080 (size=68) 2024-11-27T01:26:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741902_1080 (size=68) 2024-11-27T01:26:00,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741902_1080 (size=68) 2024-11-27T01:26:00,270 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:00,272 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,273 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName a 2024-11-27T01:26:00,273 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,273 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,274 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,274 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName b 2024-11-27T01:26:00,275 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,275 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,275 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,276 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName c 2024-11-27T01:26:00,276 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,277 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,277 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,278 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,278 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,279 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,279 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,280 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:00,281 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,283 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:26:00,283 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf7b4102551d01886a513742a3c1c7d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62539084, jitterRate=-0.06809502840042114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:00,284 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf7b4102551d01886a513742a3c1c7d9: Writing region info on filesystem at 1732670760271Initializing all the Stores at 1732670760271Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760271Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760271Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760271Cleaning up temporary data from old regions at 1732670760279 (+8 ms)Region opened successfully at 1732670760284 (+5 ms) 2024-11-27T01:26:00,285 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bf7b4102551d01886a513742a3c1c7d9, disabling compactions & flushes 2024-11-27T01:26:00,285 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:00,285 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:00,285 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. after waiting 0 ms 2024-11-27T01:26:00,285 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:00,285 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:00,285 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bf7b4102551d01886a513742a3c1c7d9: Waiting for close lock at 1732670760284Disabling compacts and flushes for region at 1732670760284Disabling writes for close at 1732670760285 (+1 ms)Writing region close event to WAL at 1732670760285Closed at 1732670760285 2024-11-27T01:26:00,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741901_1079 (size=175) 2024-11-27T01:26:00,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741901_1079 (size=175) 2024-11-27T01:26:00,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741901_1079 (size=175) 2024-11-27T01:26:00,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:26:00,291 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08745447:(num 1732670760217) 2024-11-27T01:26:00,292 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:00,293 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:00,306 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293, exclude list is [], retry=0 2024-11-27T01:26:00,309 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:00,309 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:00,309 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:00,335 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 2024-11-27T01:26:00,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:26:00,387 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bf7b4102551d01886a513742a3c1c7d9, NAME => 'testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:00,390 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,390 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:00,390 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,390 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,392 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,393 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName a 2024-11-27T01:26:00,393 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,394 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,394 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,395 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName b 2024-11-27T01:26:00,395 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,395 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,395 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,396 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName c 2024-11-27T01:26:00,396 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:00,397 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:00,397 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,397 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,398 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,399 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,399 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,400 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:00,401 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,402 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf7b4102551d01886a513742a3c1c7d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67296117, jitterRate=0.002790287137031555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:00,402 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:00,403 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf7b4102551d01886a513742a3c1c7d9: Running coprocessor pre-open hook at 1732670760390Writing region info on filesystem at 1732670760390Initializing all the Stores at 1732670760391 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760391Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760392 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670760392Cleaning up temporary data from old regions at 1732670760399 (+7 ms)Running coprocessor post-open hooks at 1732670760402 (+3 ms)Region opened successfully at 1732670760402 2024-11-27T01:26:00,413 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bf7b4102551d01886a513742a3c1c7d9 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-27T01:26:00,414 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:01,415 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:01,563 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:26:02,415 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:03,416 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:04,417 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:05,418 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:06,418 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:07,419 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:08,140 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T01:26:08,420 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:09,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-27T01:26:09,386 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-27T01:26:09,420 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bf7b4102551d01886a513742a3c1c7d9/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:09,422 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bf7b4102551d01886a513742a3c1c7d9: 2024-11-27T01:26:09,422 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bf7b4102551d01886a513742a3c1c7d9: 2024-11-27T01:26:09,437 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-27T01:26:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bf7b4102551d01886a513742a3c1c7d9, disabling compactions & flushes 2024-11-27T01:26:09,437 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. after waiting 0 ms 2024-11-27T01:26:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:09,438 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:09,438 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. 2024-11-27T01:26:09,438 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bf7b4102551d01886a513742a3c1c7d9: Waiting for close lock at 1732670769437Running coprocessor pre-close hooks at 1732670769437Disabling compacts and flushes for region at 1732670769437Disabling writes for close at 1732670769437Writing region close event to WAL at 1732670769438 (+1 ms)Running coprocessor post-close hooks at 1732670769438Closed at 1732670769438 2024-11-27T01:26:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741903_1081 (size=4236) 2024-11-27T01:26:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741903_1081 (size=4236) 2024-11-27T01:26:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741903_1081 (size=4236) 2024-11-27T01:26:09,458 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293, size=4.1 K (4236bytes) 2024-11-27T01:26:09,458 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 2024-11-27T01:26:09,459 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 after 1ms 2024-11-27T01:26:09,493 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:09,493 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 took 35ms 2024-11-27T01:26:09,496 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 so closing down 2024-11-27T01:26:09,496 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:26:09,497 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732670760293.temp 2024-11-27T01:26:09,524 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000004-wal.1732670760293.temp 2024-11-27T01:26:09,524 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:26:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741904_1082 (size=3245) 2024-11-27T01:26:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741904_1082 (size=3245) 2024-11-27T01:26:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741904_1082 (size=3245) 2024-11-27T01:26:09,532 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000004-wal.1732670760293.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-27T01:26:09,533 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000004-wal.1732670760293.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000026 2024-11-27T01:26:09,533 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-27T01:26:09,534 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293, journal: Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293, size=4.1 K (4236bytes) at 1732670769458Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 so closing down at 1732670769496 (+38 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000004-wal.1732670760293.temp at 1732670769524 (+28 ms)3 split writer threads finished at 1732670769524Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000004-wal.1732670760293.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732670769532 (+8 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000004-wal.1732670760293.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000026 at 1732670769533 (+1 ms)Processed 23 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732670769533 2024-11-27T01:26:09,535 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670760293 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670760293 2024-11-27T01:26:09,536 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000026 2024-11-27T01:26:09,536 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:09,538 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:09,551 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670769538, exclude list is [], retry=0 2024-11-27T01:26:09,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:09,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:09,555 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:09,583 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670769538 2024-11-27T01:26:09,583 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:26:09,583 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bf7b4102551d01886a513742a3c1c7d9, NAME => 'testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:09,584 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,584 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:09,584 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,584 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,586 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,587 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName a 2024-11-27T01:26:09,587 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:09,587 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:09,588 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,588 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName b 2024-11-27T01:26:09,588 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:09,589 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:09,589 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,590 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf7b4102551d01886a513742a3c1c7d9 columnFamilyName c 2024-11-27T01:26:09,590 DEBUG [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:09,590 INFO [StoreOpener-bf7b4102551d01886a513742a3c1c7d9-1 {}] regionserver.HStore(327): Store=bf7b4102551d01886a513742a3c1c7d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:09,590 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,592 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,593 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000026 2024-11-27T01:26:09,622 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:09,624 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000026 2024-11-27T01:26:09,625 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bf7b4102551d01886a513742a3c1c7d9 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-27T01:26:09,645 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/a/332945657d0843af9f97447279ea787e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732670769427/Put/seqid=0 2024-11-27T01:26:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741906_1084 (size=5523) 2024-11-27T01:26:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741906_1084 (size=5523) 2024-11-27T01:26:09,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741906_1084 (size=5523) 2024-11-27T01:26:09,655 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/a/332945657d0843af9f97447279ea787e 2024-11-27T01:26:09,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/b/64b293c05063404288f2ae4fa2bc2a1a is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732670769422/Put/seqid=0 2024-11-27T01:26:09,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741907_1085 (size=5524) 2024-11-27T01:26:09,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741907_1085 (size=5524) 2024-11-27T01:26:09,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741907_1085 (size=5524) 2024-11-27T01:26:09,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/b/64b293c05063404288f2ae4fa2bc2a1a 2024-11-27T01:26:09,710 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/c/4608171d26e840468c8cfb91713d770e is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732670769425/Put/seqid=0 2024-11-27T01:26:09,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741908_1086 (size=5457) 2024-11-27T01:26:09,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741908_1086 (size=5457) 2024-11-27T01:26:09,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741908_1086 (size=5457) 2024-11-27T01:26:09,717 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/c/4608171d26e840468c8cfb91713d770e 2024-11-27T01:26:09,723 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/a/332945657d0843af9f97447279ea787e as hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/a/332945657d0843af9f97447279ea787e 2024-11-27T01:26:09,729 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/a/332945657d0843af9f97447279ea787e, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T01:26:09,730 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/b/64b293c05063404288f2ae4fa2bc2a1a as hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/b/64b293c05063404288f2ae4fa2bc2a1a 2024-11-27T01:26:09,736 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/b/64b293c05063404288f2ae4fa2bc2a1a, entries=7, sequenceid=26, filesize=5.4 K 2024-11-27T01:26:09,737 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/.tmp/c/4608171d26e840468c8cfb91713d770e as hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/c/4608171d26e840468c8cfb91713d770e 2024-11-27T01:26:09,742 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/c/4608171d26e840468c8cfb91713d770e, entries=6, sequenceid=26, filesize=5.3 K 2024-11-27T01:26:09,743 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bf7b4102551d01886a513742a3c1c7d9 in 118ms, sequenceid=26, compaction requested=false; wal=null 2024-11-27T01:26:09,743 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/0000000000000000026 2024-11-27T01:26:09,744 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,744 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,745 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:09,746 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,748 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsAfterAbortingFlush/bf7b4102551d01886a513742a3c1c7d9/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-27T01:26:09,749 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf7b4102551d01886a513742a3c1c7d9; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71511192, jitterRate=0.06559979915618896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:09,749 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf7b4102551d01886a513742a3c1c7d9 2024-11-27T01:26:09,750 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf7b4102551d01886a513742a3c1c7d9: Running coprocessor pre-open hook at 1732670769584Writing region info on filesystem at 1732670769584Initializing all the Stores at 1732670769585 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670769585Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670769585Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670769585Obtaining lock to block concurrent updates at 1732670769625 (+40 ms)Preparing flush snapshotting stores in bf7b4102551d01886a513742a3c1c7d9 at 1732670769625Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732670769625Flushing stores of testReplayEditsAfterAbortingFlush,,1732670760213.bf7b4102551d01886a513742a3c1c7d9. at 1732670769625Flushing bf7b4102551d01886a513742a3c1c7d9/a: creating writer at 1732670769625Flushing bf7b4102551d01886a513742a3c1c7d9/a: appending metadata at 1732670769644 (+19 ms)Flushing bf7b4102551d01886a513742a3c1c7d9/a: closing flushed file at 1732670769644Flushing bf7b4102551d01886a513742a3c1c7d9/b: creating writer at 1732670769660 (+16 ms)Flushing bf7b4102551d01886a513742a3c1c7d9/b: appending metadata at 1732670769675 (+15 ms)Flushing bf7b4102551d01886a513742a3c1c7d9/b: closing flushed file at 1732670769675Flushing bf7b4102551d01886a513742a3c1c7d9/c: creating writer at 1732670769688 (+13 ms)Flushing bf7b4102551d01886a513742a3c1c7d9/c: appending metadata at 1732670769709 (+21 ms)Flushing bf7b4102551d01886a513742a3c1c7d9/c: closing flushed file at 1732670769709Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ad601b3: reopening flushed file at 1732670769722 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b00e5f: reopening flushed file at 1732670769730 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cfed60b: reopening flushed file at 1732670769736 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bf7b4102551d01886a513742a3c1c7d9 in 118ms, sequenceid=26, compaction requested=false; wal=null at 1732670769743 (+7 ms)Cleaning up temporary data from old regions at 1732670769744 (+1 ms)Running coprocessor post-open hooks at 1732670769749 (+5 ms)Region opened successfully at 1732670769750 (+1 ms) 2024-11-27T01:26:09,770 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 415), OpenFileDescriptor=1180 (was 1123) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=6209 (was 6200) - AvailableMemoryMB LEAK? - 2024-11-27T01:26:09,770 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-27T01:26:09,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=6208 2024-11-27T01:26:09,783 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-27T01:26:09,799 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:09,801 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:09,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:26:09,805 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81099440, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-81099440, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:09,826 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81099440/hregion-81099440.1732670769805, exclude list is [], retry=0 2024-11-27T01:26:09,829 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:09,829 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:09,832 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:09,869 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81099440/hregion-81099440.1732670769805 2024-11-27T01:26:09,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:26:09,870 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ee932699b1f3f747f8078482906812fb, NAME => 'testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:26:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741910_1088 (size=61) 2024-11-27T01:26:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741910_1088 (size=61) 2024-11-27T01:26:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741910_1088 (size=61) 2024-11-27T01:26:09,880 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:09,881 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,883 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee932699b1f3f747f8078482906812fb columnFamilyName a 2024-11-27T01:26:09,883 DEBUG [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:09,883 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(327): Store=ee932699b1f3f747f8078482906812fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:09,883 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,884 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,884 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,884 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,884 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,886 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,888 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:26:09,889 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee932699b1f3f747f8078482906812fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62791430, jitterRate=-0.06433477997779846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:26:09,889 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee932699b1f3f747f8078482906812fb: Writing region info on filesystem at 1732670769880Initializing all the Stores at 1732670769881 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670769881Cleaning up temporary data from old regions at 1732670769885 (+4 ms)Region opened successfully at 1732670769889 (+4 ms) 2024-11-27T01:26:09,889 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee932699b1f3f747f8078482906812fb, disabling compactions & flushes 2024-11-27T01:26:09,889 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,889 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,890 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. after waiting 0 ms 2024-11-27T01:26:09,890 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,890 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,890 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee932699b1f3f747f8078482906812fb: Waiting for close lock at 1732670769889Disabling compacts and flushes for region at 1732670769889Disabling writes for close at 1732670769890 (+1 ms)Writing region close event to WAL at 1732670769890Closed at 1732670769890 2024-11-27T01:26:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741909_1087 (size=175) 2024-11-27T01:26:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741909_1087 (size=175) 2024-11-27T01:26:09,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741909_1087 (size=175) 2024-11-27T01:26:09,895 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:26:09,895 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81099440:(num 1732670769805) 2024-11-27T01:26:09,896 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:09,897 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:09,911 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898, exclude list is [], retry=0 2024-11-27T01:26:09,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:09,914 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:09,915 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:09,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 2024-11-27T01:26:09,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:26:09,945 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee932699b1f3f747f8078482906812fb, NAME => 'testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:09,945 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:09,945 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,945 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,946 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,947 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee932699b1f3f747f8078482906812fb columnFamilyName a 2024-11-27T01:26:09,947 DEBUG [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:09,948 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(327): Store=ee932699b1f3f747f8078482906812fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:09,948 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,950 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,950 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,952 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:09,953 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee932699b1f3f747f8078482906812fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67523884, jitterRate=0.006184279918670654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:26:09,954 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee932699b1f3f747f8078482906812fb: Writing region info on filesystem at 1732670769945Initializing all the Stores at 1732670769946 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670769946Cleaning up temporary data from old regions at 1732670769950 (+4 ms)Region opened successfully at 1732670769954 (+4 ms) 2024-11-27T01:26:09,963 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ee932699b1f3f747f8078482906812fb, disabling compactions & flushes 2024-11-27T01:26:09,964 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,964 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,964 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. after waiting 0 ms 2024-11-27T01:26:09,964 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,964 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,964 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. 2024-11-27T01:26:09,964 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ee932699b1f3f747f8078482906812fb: Waiting for close lock at 1732670769963Disabling compacts and flushes for region at 1732670769963Disabling writes for close at 1732670769964 (+1 ms)Writing region close event to WAL at 1732670769964Closed at 1732670769964 2024-11-27T01:26:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741911_1089 (size=1795) 2024-11-27T01:26:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741911_1089 (size=1795) 2024-11-27T01:26:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741911_1089 (size=1795) 2024-11-27T01:26:09,984 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898, size=1.8 K (1795bytes) 2024-11-27T01:26:09,984 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 2024-11-27T01:26:09,985 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 after 1ms 2024-11-27T01:26:10,014 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:10,014 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 took 30ms 2024-11-27T01:26:10,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 so closing down 2024-11-27T01:26:10,016 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:26:10,018 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670769898.temp 2024-11-27T01:26:10,046 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000003-wal.1732670769898.temp 2024-11-27T01:26:10,046 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:26:10,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741912_1090 (size=1795) 2024-11-27T01:26:10,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741912_1090 (size=1795) 2024-11-27T01:26:10,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741912_1090 (size=1795) 2024-11-27T01:26:10,080 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000003-wal.1732670769898.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-27T01:26:10,082 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000003-wal.1732670769898.temp to hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000012 2024-11-27T01:26:10,082 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-27T01:26:10,082 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898, journal: Splitting hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898, size=1.8 K (1795bytes) at 1732670769984Finishing writing output for hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 so closing down at 1732670770016 (+32 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000003-wal.1732670769898.temp at 1732670770046 (+30 ms)3 split writer threads finished at 1732670770046Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000003-wal.1732670769898.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732670770080 (+34 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000003-wal.1732670769898.temp to hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000012 at 1732670770082 (+2 ms)Processed 10 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732670770082 2024-11-27T01:26:10,084 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670769898 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670769898 2024-11-27T01:26:10,085 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000012 2024-11-27T01:26:10,114 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:10,411 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:10,414 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:10,428 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670770414, exclude list is [], retry=0 2024-11-27T01:26:10,430 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:10,431 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:10,431 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:10,458 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670770414 2024-11-27T01:26:10,459 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:26:10,459 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee932699b1f3f747f8078482906812fb, NAME => 'testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:10,459 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:10,459 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,459 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,462 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,464 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee932699b1f3f747f8078482906812fb columnFamilyName a 2024-11-27T01:26:10,464 DEBUG [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,465 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(327): Store=ee932699b1f3f747f8078482906812fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,465 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,466 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,467 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,468 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000012 2024-11-27T01:26:10,495 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:10,497 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000012 2024-11-27T01:26:10,497 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ee932699b1f3f747f8078482906812fb 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-27T01:26:10,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/.tmp/a/17f831d6f3ca462da96bd2cd4d2eafc6 is 79, key is testDatalossWhenInputError/a:x0/1732670769954/Put/seqid=0 2024-11-27T01:26:10,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741914_1092 (size=5808) 2024-11-27T01:26:10,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741914_1092 (size=5808) 2024-11-27T01:26:10,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741914_1092 (size=5808) 2024-11-27T01:26:10,524 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/.tmp/a/17f831d6f3ca462da96bd2cd4d2eafc6 2024-11-27T01:26:10,533 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/.tmp/a/17f831d6f3ca462da96bd2cd4d2eafc6 as hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/a/17f831d6f3ca462da96bd2cd4d2eafc6 2024-11-27T01:26:10,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/a/17f831d6f3ca462da96bd2cd4d2eafc6, entries=10, sequenceid=12, filesize=5.7 K 2024-11-27T01:26:10,543 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ee932699b1f3f747f8078482906812fb in 46ms, sequenceid=12, compaction requested=false; wal=null 2024-11-27T01:26:10,544 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/0000000000000000012 2024-11-27T01:26:10,544 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,544 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,547 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,550 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-27T01:26:10,550 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee932699b1f3f747f8078482906812fb; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64304445, jitterRate=-0.04178909957408905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:26:10,551 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee932699b1f3f747f8078482906812fb: Writing region info on filesystem at 1732670770459Initializing all the Stores at 1732670770462 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770462Obtaining lock to block concurrent updates at 1732670770497 (+35 ms)Preparing flush snapshotting stores in ee932699b1f3f747f8078482906812fb at 1732670770497Finished memstore snapshotting testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732670770497Flushing stores of testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb. at 1732670770497Flushing ee932699b1f3f747f8078482906812fb/a: creating writer at 1732670770497Flushing ee932699b1f3f747f8078482906812fb/a: appending metadata at 1732670770515 (+18 ms)Flushing ee932699b1f3f747f8078482906812fb/a: closing flushed file at 1732670770515Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77948c88: reopening flushed file at 1732670770532 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ee932699b1f3f747f8078482906812fb in 46ms, sequenceid=12, compaction requested=false; wal=null at 1732670770543 (+11 ms)Cleaning up temporary data from old regions at 1732670770544 (+1 ms)Region opened successfully at 1732670770551 (+7 ms) 2024-11-27T01:26:10,554 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ee932699b1f3f747f8078482906812fb, NAME => 'testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:10,554 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732670769800.ee932699b1f3f747f8078482906812fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:10,554 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,554 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,556 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,556 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee932699b1f3f747f8078482906812fb columnFamilyName a 2024-11-27T01:26:10,557 DEBUG [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,563 DEBUG [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/a/17f831d6f3ca462da96bd2cd4d2eafc6 2024-11-27T01:26:10,563 INFO [StoreOpener-ee932699b1f3f747f8078482906812fb-1 {}] regionserver.HStore(327): Store=ee932699b1f3f747f8078482906812fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,564 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,564 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,566 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,566 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,568 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ee932699b1f3f747f8078482906812fb 2024-11-27T01:26:10,571 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testDatalossWhenInputError/ee932699b1f3f747f8078482906812fb/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-27T01:26:10,572 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ee932699b1f3f747f8078482906812fb; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70628384, jitterRate=0.0524449348449707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T01:26:10,572 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ee932699b1f3f747f8078482906812fb: Writing region info on filesystem at 1732670770554Initializing all the Stores at 1732670770555 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770555Cleaning up temporary data from old regions at 1732670770566 (+11 ms)Region opened successfully at 1732670770572 (+6 ms) 2024-11-27T01:26:10,590 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:41380 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:35082 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:41270 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:35334 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:35278 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-879573077-172.17.0.2-1732670735999:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:35016 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6178 (was 6208) 2024-11-27T01:26:10,590 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-27T01:26:10,602 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=6178 2024-11-27T01:26:10,602 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-11-27T01:26:10,617 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:10,619 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:10,620 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:26:10,622 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41869939, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-41869939, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:10,635 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41869939/hregion-41869939.1732670770622, exclude list is [], retry=0 2024-11-27T01:26:10,638 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:10,638 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:10,638 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:10,666 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41869939/hregion-41869939.1732670770622 2024-11-27T01:26:10,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:26:10,667 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 370910be2bfb204c5abd2eb55ae873ab, NAME => 'testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:26:10,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741916_1094 (size=63) 2024-11-27T01:26:10,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741916_1094 (size=63) 2024-11-27T01:26:10,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741916_1094 (size=63) 2024-11-27T01:26:10,676 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:10,678 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,679 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName a 2024-11-27T01:26:10,679 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,680 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,680 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,681 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName b 2024-11-27T01:26:10,681 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,681 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,681 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,683 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName c 2024-11-27T01:26:10,683 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,683 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,683 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,684 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,685 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,686 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,686 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,686 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:10,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,690 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:26:10,690 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 370910be2bfb204c5abd2eb55ae873ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70481428, jitterRate=0.05025511980056763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:10,691 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 370910be2bfb204c5abd2eb55ae873ab: Writing region info on filesystem at 1732670770677Initializing all the Stores at 1732670770677Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770678 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770678Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770678Cleaning up temporary data from old regions at 1732670770686 (+8 ms)Region opened successfully at 1732670770691 (+5 ms) 2024-11-27T01:26:10,691 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 370910be2bfb204c5abd2eb55ae873ab, disabling compactions & flushes 2024-11-27T01:26:10,691 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:10,692 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:10,692 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. after waiting 0 ms 2024-11-27T01:26:10,692 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:10,692 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:10,692 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 370910be2bfb204c5abd2eb55ae873ab: Waiting for close lock at 1732670770691Disabling compacts and flushes for region at 1732670770691Disabling writes for close at 1732670770692 (+1 ms)Writing region close event to WAL at 1732670770692Closed at 1732670770692 2024-11-27T01:26:10,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741915_1093 (size=175) 2024-11-27T01:26:10,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741915_1093 (size=175) 2024-11-27T01:26:10,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741915_1093 (size=175) 2024-11-27T01:26:10,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:26:10,697 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41869939:(num 1732670770622) 2024-11-27T01:26:10,697 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:10,699 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:10,712 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699, exclude list is [], retry=0 2024-11-27T01:26:10,715 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:10,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:10,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:10,744 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 2024-11-27T01:26:10,744 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:26:10,745 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 370910be2bfb204c5abd2eb55ae873ab, NAME => 'testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:10,745 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:10,745 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,745 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,746 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,747 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName a 2024-11-27T01:26:10,747 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,748 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,748 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,749 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName b 2024-11-27T01:26:10,749 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,749 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,749 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,750 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName c 2024-11-27T01:26:10,750 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:10,750 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:10,750 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,752 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,752 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,753 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:10,754 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:10,755 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 370910be2bfb204c5abd2eb55ae873ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69218055, jitterRate=0.03142939507961273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:10,755 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 370910be2bfb204c5abd2eb55ae873ab: Writing region info on filesystem at 1732670770745Initializing all the Stores at 1732670770746 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770746Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770746Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670770746Cleaning up temporary data from old regions at 1732670770752 (+6 ms)Region opened successfully at 1732670770755 (+3 ms) 2024-11-27T01:26:10,759 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732670770759/Put/seqid=0 2024-11-27T01:26:10,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741918_1096 (size=4875) 2024-11-27T01:26:10,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741918_1096 (size=4875) 2024-11-27T01:26:10,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741918_1096 (size=4875) 2024-11-27T01:26:10,769 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732670770769/Put/seqid=0 2024-11-27T01:26:10,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741919_1097 (size=4875) 2024-11-27T01:26:10,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741919_1097 (size=4875) 2024-11-27T01:26:10,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741919_1097 (size=4875) 2024-11-27T01:26:10,778 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732670770778/Put/seqid=0 2024-11-27T01:26:10,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741920_1098 (size=4875) 2024-11-27T01:26:10,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741920_1098 (size=4875) 2024-11-27T01:26:10,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741920_1098 (size=4875) 2024-11-27T01:26:10,785 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 370910be2bfb204c5abd2eb55ae873ab/a 2024-11-27T01:26:10,789 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-27T01:26:10,789 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:26:10,789 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 370910be2bfb204c5abd2eb55ae873ab/a 2024-11-27T01:26:10,792 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-27T01:26:10,792 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:26:10,792 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 370910be2bfb204c5abd2eb55ae873ab/a 2024-11-27T01:26:10,795 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-27T01:26:10,795 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-27T01:26:10,795 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 370910be2bfb204c5abd2eb55ae873ab 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-27T01:26:10,811 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/.tmp/a/6beb0812b14c409b90c62e29ec7a814e is 55, key is testCompactedBulkLoadedFiles/a:a/1732670770756/Put/seqid=0 2024-11-27T01:26:10,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741921_1099 (size=5107) 2024-11-27T01:26:10,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741921_1099 (size=5107) 2024-11-27T01:26:10,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741921_1099 (size=5107) 2024-11-27T01:26:10,819 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/.tmp/a/6beb0812b14c409b90c62e29ec7a814e 2024-11-27T01:26:10,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/.tmp/a/6beb0812b14c409b90c62e29ec7a814e as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e 2024-11-27T01:26:10,830 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e, entries=1, sequenceid=4, filesize=5.0 K 2024-11-27T01:26:10,831 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 370910be2bfb204c5abd2eb55ae873ab in 36ms, sequenceid=4, compaction requested=false 2024-11-27T01:26:10,831 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 370910be2bfb204c5abd2eb55ae873ab: 2024-11-27T01:26:10,832 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ 2024-11-27T01:26:10,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ 2024-11-27T01:26:10,834 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ 2024-11-27T01:26:10,835 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile0 into 370910be2bfb204c5abd2eb55ae873ab/a as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ - updating store file list. 2024-11-27T01:26:10,839 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 325cf8a46b164b2dbf61133be734d8f6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:26:10,840 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ into 370910be2bfb204c5abd2eb55ae873ab/a 2024-11-27T01:26:10,840 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile0 into 370910be2bfb204c5abd2eb55ae873ab/a (new location: hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_) 2024-11-27T01:26:10,841 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile1 into 370910be2bfb204c5abd2eb55ae873ab/a as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ - updating store file list. 2024-11-27T01:26:10,845 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8ed1928986ea4277b8c2781e982d59c2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:26:10,846 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ into 370910be2bfb204c5abd2eb55ae873ab/a 2024-11-27T01:26:10,846 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile1 into 370910be2bfb204c5abd2eb55ae873ab/a (new location: hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_) 2024-11-27T01:26:10,847 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile2 into 370910be2bfb204c5abd2eb55ae873ab/a as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ - updating store file list. 2024-11-27T01:26:10,852 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:26:10,852 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ into 370910be2bfb204c5abd2eb55ae873ab/a 2024-11-27T01:26:10,852 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:36155/hbase/testCompactedBulkLoadedFiles/hfile2 into 370910be2bfb204c5abd2eb55ae873ab/a (new location: hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_) 2024-11-27T01:26:10,861 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T01:26:10,861 DEBUG [Time-limited test {}] regionserver.HStore(1541): 370910be2bfb204c5abd2eb55ae873ab/a is initiating major compaction (all files) 2024-11-27T01:26:10,861 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 370910be2bfb204c5abd2eb55ae873ab/a in testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:10,861 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_] into tmpdir=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/.tmp, totalSize=19.3 K 2024-11-27T01:26:10,862 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6beb0812b14c409b90c62e29ec7a814e, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732670770756 2024-11-27T01:26:10,862 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 325cf8a46b164b2dbf61133be734d8f6_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T01:26:10,863 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8ed1928986ea4277b8c2781e982d59c2_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T01:26:10,863 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-27T01:26:10,876 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/.tmp/a/a28d7f87e62f4fdabdc09588f6db9e6a is 55, key is testCompactedBulkLoadedFiles/a:a/1732670770756/Put/seqid=0 2024-11-27T01:26:10,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741922_1100 (size=6154) 2024-11-27T01:26:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741922_1100 (size=6154) 2024-11-27T01:26:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741922_1100 (size=6154) 2024-11-27T01:26:10,893 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/.tmp/a/a28d7f87e62f4fdabdc09588f6db9e6a as hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/a28d7f87e62f4fdabdc09588f6db9e6a 2024-11-27T01:26:10,900 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 370910be2bfb204c5abd2eb55ae873ab/a of 370910be2bfb204c5abd2eb55ae873ab into a28d7f87e62f4fdabdc09588f6db9e6a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T01:26:10,900 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 370910be2bfb204c5abd2eb55ae873ab: 2024-11-27T01:26:10,900 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T01:26:10,900 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-27T01:26:10,952 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699, size=0 (0bytes) 2024-11-27T01:26:10,952 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 might be still open, length is 0 2024-11-27T01:26:10,953 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 2024-11-27T01:26:10,953 WARN [IPC Server handler 2 on default port 36155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-27T01:26:10,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 after 1ms 2024-11-27T01:26:13,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:49168 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45665:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49168 dst: /127.0.0.1:45665 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45665 remote=/127.0.0.1:49168]. Total timeout mills is 60000, 57026 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:13,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57316 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:44005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57316 dst: /127.0.0.1:44005 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:13,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:41310 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41310 dst: /127.0.0.1:39261 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741917_1101 (size=1794) 2024-11-27T01:26:13,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741917_1101 (size=1794) 2024-11-27T01:26:14,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 after 4001ms 2024-11-27T01:26:14,958 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T01:26:14,983 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:14,983 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 took 4031ms 2024-11-27T01:26:14,985 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699; continuing. 2024-11-27T01:26:14,985 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 so closing down 2024-11-27T01:26:14,985 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:26:14,986 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670770699.temp 2024-11-27T01:26:15,012 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000003-wal.1732670770699.temp 2024-11-27T01:26:15,012 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:26:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741923_1102 (size=800) 2024-11-27T01:26:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741923_1102 (size=800) 2024-11-27T01:26:15,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741923_1102 (size=800) 2024-11-27T01:26:15,019 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000003-wal.1732670770699.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-27T01:26:15,020 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000003-wal.1732670770699.temp to hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000008 2024-11-27T01:26:15,020 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:26:15,021 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699, journal: Splitting hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699, size=0 (0bytes) at 1732670770952Finishing writing output for hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 so closing down at 1732670774985 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000003-wal.1732670770699.temp at 1732670775012 (+27 ms)3 split writer threads finished at 1732670775012Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000003-wal.1732670770699.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732670775019 (+7 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000003-wal.1732670770699.temp to hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000008 at 1732670775020 (+1 ms)Processed 5 edits across 1 Regions in 37 ms; skipped=3; WAL=hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699, size=0, length=0, corrupted=false, cancelled=false at 1732670775020 2024-11-27T01:26:15,022 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670770699 2024-11-27T01:26:15,023 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000008 2024-11-27T01:26:15,023 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:15,024 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:15,037 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670775025, exclude list is [], retry=0 2024-11-27T01:26:15,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:15,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:15,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:15,068 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670775025 2024-11-27T01:26:15,068 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:26:15,068 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 370910be2bfb204c5abd2eb55ae873ab, NAME => 'testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:15,068 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:15,069 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,069 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,070 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,071 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName a 2024-11-27T01:26:15,071 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,077 DEBUG [StoreFileOpener-370910be2bfb204c5abd2eb55ae873ab-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 325cf8a46b164b2dbf61133be734d8f6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:26:15,077 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ 2024-11-27T01:26:15,081 DEBUG [StoreFileOpener-370910be2bfb204c5abd2eb55ae873ab-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:26:15,081 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ 2024-11-27T01:26:15,084 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e 2024-11-27T01:26:15,087 DEBUG [StoreFileOpener-370910be2bfb204c5abd2eb55ae873ab-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8ed1928986ea4277b8c2781e982d59c2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-27T01:26:15,087 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ 2024-11-27T01:26:15,090 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/a28d7f87e62f4fdabdc09588f6db9e6a 2024-11-27T01:26:15,090 WARN [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16af31bc 2024-11-27T01:26:15,090 WARN [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16af31bc 2024-11-27T01:26:15,090 WARN [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16af31bc 2024-11-27T01:26:15,090 WARN [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16af31bc 2024-11-27T01:26:15,091 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_] to archive 2024-11-27T01:26:15,091 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T01:26:15,093 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ to hdfs://localhost:36155/hbase/archive/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/325cf8a46b164b2dbf61133be734d8f6_SeqId_4_ 2024-11-27T01:26:15,094 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ to hdfs://localhost:36155/hbase/archive/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_ 2024-11-27T01:26:15,095 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e to hdfs://localhost:36155/hbase/archive/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/6beb0812b14c409b90c62e29ec7a814e 2024-11-27T01:26:15,096 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ to hdfs://localhost:36155/hbase/archive/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/a/8ed1928986ea4277b8c2781e982d59c2_SeqId_4_ 2024-11-27T01:26:15,096 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,096 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,097 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName b 2024-11-27T01:26:15,097 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,098 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,098 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,098 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 370910be2bfb204c5abd2eb55ae873ab columnFamilyName c 2024-11-27T01:26:15,098 DEBUG [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,099 INFO [StoreOpener-370910be2bfb204c5abd2eb55ae873ab-1 {}] regionserver.HStore(327): Store=370910be2bfb204c5abd2eb55ae873ab/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,099 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,100 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,101 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,101 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000008 2024-11-27T01:26:15,128 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:15,130 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 370910be2bfb204c5abd2eb55ae873ab : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "370910be2bfb204c5abd2eb55ae873ab" family_name: "a" compaction_input: "6beb0812b14c409b90c62e29ec7a814e" compaction_input: "325cf8a46b164b2dbf61133be734d8f6_SeqId_4_" compaction_input: "8ed1928986ea4277b8c2781e982d59c2_SeqId_4_" compaction_input: "5f95c2f6ddc44a4690e6652019a34c3f_SeqId_4_" compaction_output: "a28d7f87e62f4fdabdc09588f6db9e6a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-27T01:26:15,130 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-27T01:26:15,130 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000008 2024-11-27T01:26:15,131 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/0000000000000000008 2024-11-27T01:26:15,132 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,132 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,133 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:15,134 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 370910be2bfb204c5abd2eb55ae873ab 2024-11-27T01:26:15,136 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testCompactedBulkLoadedFiles/370910be2bfb204c5abd2eb55ae873ab/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-27T01:26:15,137 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 370910be2bfb204c5abd2eb55ae873ab; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63196107, jitterRate=-0.058304622769355774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:15,137 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 370910be2bfb204c5abd2eb55ae873ab: Writing region info on filesystem at 1732670775069Initializing all the Stores at 1732670775070 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775070Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775070Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775070Cleaning up temporary data from old regions at 1732670775132 (+62 ms)Region opened successfully at 1732670775137 (+5 ms) 2024-11-27T01:26:15,139 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 370910be2bfb204c5abd2eb55ae873ab, disabling compactions & flushes 2024-11-27T01:26:15,139 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:15,139 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:15,139 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. after waiting 0 ms 2024-11-27T01:26:15,139 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:15,140 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732670770618.370910be2bfb204c5abd2eb55ae873ab. 2024-11-27T01:26:15,140 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 370910be2bfb204c5abd2eb55ae873ab: Waiting for close lock at 1732670775139Disabling compacts and flushes for region at 1732670775139Disabling writes for close at 1732670775139Writing region close event to WAL at 1732670775140 (+1 ms)Closed at 1732670775140 2024-11-27T01:26:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741924_1103 (size=175) 2024-11-27T01:26:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741924_1103 (size=175) 2024-11-27T01:26:15,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741924_1103 (size=175) 2024-11-27T01:26:15,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:26:15,144 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670775025) 2024-11-27T01:26:15,160 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=440 (was 425) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1202150484_22 at /127.0.0.1:41358 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1202150484_22 at /127.0.0.1:49250 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34577 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1202150484_22 at /127.0.0.1:57386 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:36155 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:34577 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:36155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=6167 (was 6178) 2024-11-27T01:26:15,160 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-27T01:26:15,173 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=6166 2024-11-27T01:26:15,173 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-27T01:26:15,188 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:15,189 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T01:26:15,190 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T01:26:15,192 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23264699, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/hregion-23264699, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:15,204 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23264699/hregion-23264699.1732670775192, exclude list is [], retry=0 2024-11-27T01:26:15,207 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:15,207 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:15,207 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:15,234 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23264699/hregion-23264699.1732670775192 2024-11-27T01:26:15,234 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:26:15,234 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28967a5c265efff164e96a1412f9bd68, NAME => 'testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36155/hbase 2024-11-27T01:26:15,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741926_1105 (size=67) 2024-11-27T01:26:15,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741926_1105 (size=67) 2024-11-27T01:26:15,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741926_1105 (size=67) 2024-11-27T01:26:15,243 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:15,244 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,246 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName a 2024-11-27T01:26:15,246 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,246 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,246 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,248 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName b 2024-11-27T01:26:15,248 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,248 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,248 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,249 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName c 2024-11-27T01:26:15,249 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,250 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,250 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,250 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,251 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,252 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,252 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,252 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:15,254 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,257 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T01:26:15,257 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28967a5c265efff164e96a1412f9bd68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64246226, jitterRate=-0.04265663027763367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:15,258 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28967a5c265efff164e96a1412f9bd68: Writing region info on filesystem at 1732670775243Initializing all the Stores at 1732670775244 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775244Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775244Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775244Cleaning up temporary data from old regions at 1732670775252 (+8 ms)Region opened successfully at 1732670775257 (+5 ms) 2024-11-27T01:26:15,258 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28967a5c265efff164e96a1412f9bd68, disabling compactions & flushes 2024-11-27T01:26:15,258 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,258 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,258 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. after waiting 0 ms 2024-11-27T01:26:15,258 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,258 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,258 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28967a5c265efff164e96a1412f9bd68: Waiting for close lock at 1732670775258Disabling compacts and flushes for region at 1732670775258Disabling writes for close at 1732670775258Writing region close event to WAL at 1732670775258Closed at 1732670775258 2024-11-27T01:26:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741925_1104 (size=175) 2024-11-27T01:26:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741925_1104 (size=175) 2024-11-27T01:26:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741925_1104 (size=175) 2024-11-27T01:26:15,263 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:26:15,263 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23264699:(num 1732670775192) 2024-11-27T01:26:15,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:15,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:15,278 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265, exclude list is [], retry=0 2024-11-27T01:26:15,280 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:15,281 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:15,281 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:15,307 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 2024-11-27T01:26:15,307 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391)] 2024-11-27T01:26:15,307 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28967a5c265efff164e96a1412f9bd68, NAME => 'testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:15,308 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:15,308 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,308 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,309 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,310 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName a 2024-11-27T01:26:15,310 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,311 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,311 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,311 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName b 2024-11-27T01:26:15,311 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,312 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,312 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,313 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName c 2024-11-27T01:26:15,313 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,313 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,313 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,314 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,315 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,316 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,316 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,316 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:15,318 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28967a5c265efff164e96a1412f9bd68; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62141251, jitterRate=-0.07402320206165314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:15,318 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28967a5c265efff164e96a1412f9bd68: Writing region info on filesystem at 1732670775308Initializing all the Stores at 1732670775309 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775309Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775309Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775309Cleaning up temporary data from old regions at 1732670775316 (+7 ms)Region opened successfully at 1732670775318 (+2 ms) 2024-11-27T01:26:15,327 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28967a5c265efff164e96a1412f9bd68 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-27T01:26:15,348 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/a/20d82196c08f434e9597c012622a92e7 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732670775318/Put/seqid=0 2024-11-27T01:26:15,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741928_1107 (size=5958) 2024-11-27T01:26:15,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741928_1107 (size=5958) 2024-11-27T01:26:15,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741928_1107 (size=5958) 2024-11-27T01:26:15,355 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/a/20d82196c08f434e9597c012622a92e7 2024-11-27T01:26:15,361 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/a/20d82196c08f434e9597c012622a92e7 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/a/20d82196c08f434e9597c012622a92e7 2024-11-27T01:26:15,365 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/a/20d82196c08f434e9597c012622a92e7, entries=10, sequenceid=13, filesize=5.8 K 2024-11-27T01:26:15,366 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 28967a5c265efff164e96a1412f9bd68 in 39ms, sequenceid=13, compaction requested=false 2024-11-27T01:26:15,366 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28967a5c265efff164e96a1412f9bd68: 2024-11-27T01:26:15,382 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28967a5c265efff164e96a1412f9bd68, disabling compactions & flushes 2024-11-27T01:26:15,382 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,382 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,382 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. after waiting 0 ms 2024-11-27T01:26:15,382 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,383 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,383 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:15,383 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28967a5c265efff164e96a1412f9bd68: Waiting for close lock at 1732670775382Disabling compacts and flushes for region at 1732670775382Disabling writes for close at 1732670775382Writing region close event to WAL at 1732670775383 (+1 ms)Closed at 1732670775383 2024-11-27T01:26:15,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741927_1106 (size=6243) 2024-11-27T01:26:15,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741927_1106 (size=6243) 2024-11-27T01:26:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741927_1106 (size=6243) 2024-11-27T01:26:15,401 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265, size=6.1 K (6243bytes) 2024-11-27T01:26:15,401 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 2024-11-27T01:26:15,402 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 after 1ms 2024-11-27T01:26:15,430 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:15,430 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 took 29ms 2024-11-27T01:26:15,432 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 so closing down 2024-11-27T01:26:15,433 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:26:15,434 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732670775265.temp 2024-11-27T01:26:15,460 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000003-wal.1732670775265.temp 2024-11-27T01:26:15,461 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:26:15,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741929_1108 (size=5575) 2024-11-27T01:26:15,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741929_1108 (size=5575) 2024-11-27T01:26:15,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741929_1108 (size=5575) 2024-11-27T01:26:15,469 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000003-wal.1732670775265.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T01:26:15,470 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000003-wal.1732670775265.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000035 2024-11-27T01:26:15,471 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-27T01:26:15,471 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265, journal: Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265, size=6.1 K (6243bytes) at 1732670775401Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 so closing down at 1732670775432 (+31 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000003-wal.1732670775265.temp at 1732670775460 (+28 ms)3 split writer threads finished at 1732670775461 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000003-wal.1732670775265.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732670775469 (+8 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000003-wal.1732670775265.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000035 at 1732670775470 (+1 ms)Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732670775471 (+1 ms) 2024-11-27T01:26:15,472 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775265 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670775265 2024-11-27T01:26:15,473 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000035 2024-11-27T01:26:15,473 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:15,475 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:15,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475, exclude list is [], retry=0 2024-11-27T01:26:15,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:15,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:15,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:15,518 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 2024-11-27T01:26:15,518 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35465:35465),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:41833:41833)] 2024-11-27T01:26:15,518 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28967a5c265efff164e96a1412f9bd68, NAME => 'testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.', STARTKEY => '', ENDKEY => ''} 2024-11-27T01:26:15,519 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:15,519 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,519 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,520 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,521 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName a 2024-11-27T01:26:15,521 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,526 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/a/20d82196c08f434e9597c012622a92e7 2024-11-27T01:26:15,527 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,527 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,528 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName b 2024-11-27T01:26:15,528 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,528 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,528 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,529 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName c 2024-11-27T01:26:15,529 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:15,529 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:15,529 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,532 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000035 2024-11-27T01:26:15,559 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:15,560 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000035 2024-11-27T01:26:15,560 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28967a5c265efff164e96a1412f9bd68 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-27T01:26:15,576 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/b/7a8da738198c49ef93ec0713e8bbfc9b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732670775366/Put/seqid=0 2024-11-27T01:26:15,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741931_1110 (size=5958) 2024-11-27T01:26:15,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741931_1110 (size=5958) 2024-11-27T01:26:15,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741931_1110 (size=5958) 2024-11-27T01:26:15,584 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/b/7a8da738198c49ef93ec0713e8bbfc9b 2024-11-27T01:26:15,604 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/c/2bfc396f7d5a44248c29cae6ab12e8c3 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732670775373/Put/seqid=0 2024-11-27T01:26:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741932_1111 (size=5958) 2024-11-27T01:26:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741932_1111 (size=5958) 2024-11-27T01:26:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741932_1111 (size=5958) 2024-11-27T01:26:15,611 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/c/2bfc396f7d5a44248c29cae6ab12e8c3 2024-11-27T01:26:15,618 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/b/7a8da738198c49ef93ec0713e8bbfc9b as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/b/7a8da738198c49ef93ec0713e8bbfc9b 2024-11-27T01:26:15,624 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/b/7a8da738198c49ef93ec0713e8bbfc9b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T01:26:15,625 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/c/2bfc396f7d5a44248c29cae6ab12e8c3 as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/c/2bfc396f7d5a44248c29cae6ab12e8c3 2024-11-27T01:26:15,631 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/c/2bfc396f7d5a44248c29cae6ab12e8c3, entries=10, sequenceid=35, filesize=5.8 K 2024-11-27T01:26:15,631 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 28967a5c265efff164e96a1412f9bd68 in 71ms, sequenceid=35, compaction requested=false; wal=null 2024-11-27T01:26:15,632 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000035 2024-11-27T01:26:15,633 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,633 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,634 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:15,635 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:15,638 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-27T01:26:15,639 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28967a5c265efff164e96a1412f9bd68; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75385273, jitterRate=0.12332810461521149}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:15,639 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28967a5c265efff164e96a1412f9bd68: Writing region info on filesystem at 1732670775519Initializing all the Stores at 1732670775520 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775520Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775520Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670775520Obtaining lock to block concurrent updates at 1732670775560 (+40 ms)Preparing flush snapshotting stores in 28967a5c265efff164e96a1412f9bd68 at 1732670775560Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732670775560Flushing stores of testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. at 1732670775560Flushing 28967a5c265efff164e96a1412f9bd68/b: creating writer at 1732670775560Flushing 28967a5c265efff164e96a1412f9bd68/b: appending metadata at 1732670775575 (+15 ms)Flushing 28967a5c265efff164e96a1412f9bd68/b: closing flushed file at 1732670775575Flushing 28967a5c265efff164e96a1412f9bd68/c: creating writer at 1732670775589 (+14 ms)Flushing 28967a5c265efff164e96a1412f9bd68/c: appending metadata at 1732670775603 (+14 ms)Flushing 28967a5c265efff164e96a1412f9bd68/c: closing flushed file at 1732670775603Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63df7bd8: reopening flushed file at 1732670775617 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3448ca68: reopening flushed file at 1732670775624 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 28967a5c265efff164e96a1412f9bd68 in 71ms, sequenceid=35, compaction requested=false; wal=null at 1732670775631 (+7 ms)Cleaning up temporary data from old regions at 1732670775633 (+2 ms)Region opened successfully at 1732670775639 (+6 ms) 2024-11-27T01:26:15,707 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475, size=0 (0bytes) 2024-11-27T01:26:15,707 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 might be still open, length is 0 2024-11-27T01:26:15,707 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 2024-11-27T01:26:15,707 WARN [IPC Server handler 0 on default port 36155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-27T01:26:15,707 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 after 0ms 2024-11-27T01:26:16,956 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:57440 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:44005:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57440 dst: /127.0.0.1:44005 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44005 remote=/127.0.0.1:57440]. Total timeout mills is 60000, 58712 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:16,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:49300 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45665:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49300 dst: /127.0.0.1:45665 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:16,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-674852374_22 at /127.0.0.1:41424 [Receiving block BP-879573077-172.17.0.2-1732670735999:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41424 dst: /127.0.0.1:39261 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741930_1112 (size=5567) 2024-11-27T01:26:16,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741930_1112 (size=5567) 2024-11-27T01:26:16,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741930_1112 (size=5567) 2024-11-27T01:26:19,385 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-27T01:26:19,386 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-27T01:26:19,386 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-27T01:26:19,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-27T01:26:19,708 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 after 4001ms 2024-11-27T01:26:19,736 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:19,736 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 took 4030ms 2024-11-27T01:26:19,738 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475; continuing. 2024-11-27T01:26:19,738 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 so closing down 2024-11-27T01:26:19,738 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-27T01:26:19,739 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732670775475.temp 2024-11-27T01:26:19,765 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000037-wal.1732670775475.temp 2024-11-27T01:26:19,766 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-27T01:26:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741933_1113 (size=5575) 2024-11-27T01:26:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741933_1113 (size=5575) 2024-11-27T01:26:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741933_1113 (size=5575) 2024-11-27T01:26:19,773 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000037-wal.1732670775475.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-27T01:26:19,774 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000037-wal.1732670775475.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000066 2024-11-27T01:26:19,774 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475, size=0, length=0, corrupted=false, cancelled=false 2024-11-27T01:26:19,774 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475, journal: Splitting hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475, size=0 (0bytes) at 1732670775707Finishing writing output for hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 so closing down at 1732670779738 (+4031 ms)Creating recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000037-wal.1732670775475.temp at 1732670779765 (+27 ms)3 split writer threads finished at 1732670779766 (+1 ms)Closed recovered edits writer path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000037-wal.1732670775475.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732670779773 (+7 ms)Rename recovered edits hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000037-wal.1732670775475.temp to hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000066 at 1732670779774 (+1 ms)Processed 30 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475, size=0, length=0, corrupted=false, cancelled=false at 1732670779774 2024-11-27T01:26:19,776 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 to hdfs://localhost:36155/hbase/oldWALs/wal.1732670775475 2024-11-27T01:26:19,777 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000066 2024-11-27T01:26:19,777 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-27T01:26:19,778 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:36155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187, archiveDir=hdfs://localhost:36155/hbase/oldWALs, maxLogs=32 2024-11-27T01:26:19,791 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670779778, exclude list is [], retry=0 2024-11-27T01:26:19,793 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39261,DS-0e66b21b-4d5a-40cc-bfda-a82a348274e9,DISK] 2024-11-27T01:26:19,793 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45665,DS-61b35176-9b92-4ab0-bf15-999f3f0320fa,DISK] 2024-11-27T01:26:19,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44005,DS-514f1085-51f8-4d8d-97bc-002604feb161,DISK] 2024-11-27T01:26:19,819 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670779778 2024-11-27T01:26:19,819 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41833:41833),(127.0.0.1/127.0.0.1:38391:38391),(127.0.0.1/127.0.0.1:35465:35465)] 2024-11-27T01:26:19,820 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T01:26:19,821 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,822 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName a 2024-11-27T01:26:19,822 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:19,826 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/a/20d82196c08f434e9597c012622a92e7 2024-11-27T01:26:19,827 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:19,827 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,828 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName b 2024-11-27T01:26:19,828 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:19,832 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/b/7a8da738198c49ef93ec0713e8bbfc9b 2024-11-27T01:26:19,832 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:19,832 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,833 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28967a5c265efff164e96a1412f9bd68 columnFamilyName c 2024-11-27T01:26:19,833 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T01:26:19,837 DEBUG [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/c/2bfc396f7d5a44248c29cae6ab12e8c3 2024-11-27T01:26:19,837 INFO [StoreOpener-28967a5c265efff164e96a1412f9bd68-1 {}] regionserver.HStore(327): Store=28967a5c265efff164e96a1412f9bd68/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T01:26:19,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,840 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000066 2024-11-27T01:26:19,867 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-27T01:26:19,872 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000066 2024-11-27T01:26:19,872 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28967a5c265efff164e96a1412f9bd68 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-27T01:26:19,886 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/a/2754843fa5964a13ae197bd091b9364b is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732670775647/Put/seqid=0 2024-11-27T01:26:19,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741935_1115 (size=5958) 2024-11-27T01:26:19,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741935_1115 (size=5958) 2024-11-27T01:26:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741935_1115 (size=5958) 2024-11-27T01:26:19,893 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/a/2754843fa5964a13ae197bd091b9364b 2024-11-27T01:26:19,918 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/b/cccecf48792f4828be1cf80894bc3b3c is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732670775654/Put/seqid=0 2024-11-27T01:26:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741936_1116 (size=5958) 2024-11-27T01:26:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741936_1116 (size=5958) 2024-11-27T01:26:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741936_1116 (size=5958) 2024-11-27T01:26:19,925 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/b/cccecf48792f4828be1cf80894bc3b3c 2024-11-27T01:26:19,943 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/c/d84321d5186b4d5ba66de31149e92d8e is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732670775661/Put/seqid=0 2024-11-27T01:26:19,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741937_1117 (size=5958) 2024-11-27T01:26:19,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741937_1117 (size=5958) 2024-11-27T01:26:19,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741937_1117 (size=5958) 2024-11-27T01:26:19,950 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/c/d84321d5186b4d5ba66de31149e92d8e 2024-11-27T01:26:19,955 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/a/2754843fa5964a13ae197bd091b9364b as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/a/2754843fa5964a13ae197bd091b9364b 2024-11-27T01:26:19,960 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/a/2754843fa5964a13ae197bd091b9364b, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T01:26:19,961 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/b/cccecf48792f4828be1cf80894bc3b3c as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/b/cccecf48792f4828be1cf80894bc3b3c 2024-11-27T01:26:19,965 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/b/cccecf48792f4828be1cf80894bc3b3c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T01:26:19,965 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/.tmp/c/d84321d5186b4d5ba66de31149e92d8e as hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/c/d84321d5186b4d5ba66de31149e92d8e 2024-11-27T01:26:19,969 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/c/d84321d5186b4d5ba66de31149e92d8e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-27T01:26:19,969 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 28967a5c265efff164e96a1412f9bd68 in 97ms, sequenceid=66, compaction requested=false; wal=null 2024-11-27T01:26:19,970 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/0000000000000000066 2024-11-27T01:26:19,971 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,971 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,971 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T01:26:19,972 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28967a5c265efff164e96a1412f9bd68 2024-11-27T01:26:19,974 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/hbase/data/default/testReplayEditsWrittenViaHRegion/28967a5c265efff164e96a1412f9bd68/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-27T01:26:19,975 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28967a5c265efff164e96a1412f9bd68; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59872228, jitterRate=-0.10783427953720093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T01:26:19,975 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28967a5c265efff164e96a1412f9bd68: Writing region info on filesystem at 1732670779820Initializing all the Stores at 1732670779820Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670779820Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670779821 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732670779821Obtaining lock to block concurrent updates at 1732670779872 (+51 ms)Preparing flush snapshotting stores in 28967a5c265efff164e96a1412f9bd68 at 1732670779872Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732670779872Flushing stores of testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. at 1732670779872Flushing 28967a5c265efff164e96a1412f9bd68/a: creating writer at 1732670779872Flushing 28967a5c265efff164e96a1412f9bd68/a: appending metadata at 1732670779885 (+13 ms)Flushing 28967a5c265efff164e96a1412f9bd68/a: closing flushed file at 1732670779885Flushing 28967a5c265efff164e96a1412f9bd68/b: creating writer at 1732670779897 (+12 ms)Flushing 28967a5c265efff164e96a1412f9bd68/b: appending metadata at 1732670779917 (+20 ms)Flushing 28967a5c265efff164e96a1412f9bd68/b: closing flushed file at 1732670779917Flushing 28967a5c265efff164e96a1412f9bd68/c: creating writer at 1732670779929 (+12 ms)Flushing 28967a5c265efff164e96a1412f9bd68/c: appending metadata at 1732670779943 (+14 ms)Flushing 28967a5c265efff164e96a1412f9bd68/c: closing flushed file at 1732670779943Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a64e501: reopening flushed file at 1732670779954 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d9d8907: reopening flushed file at 1732670779960 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47a78ee4: reopening flushed file at 1732670779965 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 28967a5c265efff164e96a1412f9bd68 in 97ms, sequenceid=66, compaction requested=false; wal=null at 1732670779969 (+4 ms)Cleaning up temporary data from old regions at 1732670779971 (+2 ms)Region opened successfully at 1732670779975 (+4 ms) 2024-11-27T01:26:19,986 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28967a5c265efff164e96a1412f9bd68, disabling compactions & flushes 2024-11-27T01:26:19,986 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:19,986 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:19,986 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. after waiting 0 ms 2024-11-27T01:26:19,986 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:19,988 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732670775188.28967a5c265efff164e96a1412f9bd68. 2024-11-27T01:26:19,988 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28967a5c265efff164e96a1412f9bd68: Waiting for close lock at 1732670779986Disabling compacts and flushes for region at 1732670779986Disabling writes for close at 1732670779986Writing region close event to WAL at 1732670779988 (+2 ms)Closed at 1732670779988 2024-11-27T01:26:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741934_1114 (size=175) 2024-11-27T01:26:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741934_1114 (size=175) 2024-11-27T01:26:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741934_1114 (size=175) 2024-11-27T01:26:19,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-27T01:26:19,992 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732670779778) 2024-11-27T01:26:20,009 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=446 (was 440) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (684631574) connection to localhost/127.0.0.1:36155 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:36155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_807111476_22 at /127.0.0.1:41454 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_807111476_22 at /127.0.0.1:57482 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_807111476_22 at /127.0.0.1:49388 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=256 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=6153 (was 6166) 2024-11-27T01:26:20,009 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-27T01:26:20,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T01:26:20,009 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:26:20,009 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:26:20,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T01:26:20,010 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T01:26:20,011 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1659134806, stopped=false 2024-11-27T01:26:20,011 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7d6f26a4c136,37429,1732670738986 2024-11-27T01:26:20,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:26:20,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:26:20,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:26:20,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:26:20,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:26:20,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T01:26:20,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:26:20,014 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T01:26:20,014 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:26:20,014 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:26:20,014 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:26:20,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,014 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T01:26:20,014 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,39245,1732670739775' ***** 2024-11-27T01:26:20,014 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:26:20,014 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d6f26a4c136,43859,1732670739814' ***** 2024-11-27T01:26:20,014 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T01:26:20,015 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:26:20,015 INFO [RS:1;7d6f26a4c136:39245 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:26:20,015 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T01:26:20,015 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:26:20,015 INFO [RS:1;7d6f26a4c136:39245 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:26:20,015 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:26:20,015 INFO [RS:2;7d6f26a4c136:43859 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T01:26:20,015 INFO [RS:2;7d6f26a4c136:43859 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T01:26:20,015 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T01:26:20,016 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,43859,1732670739814 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(959): stopping server 7d6f26a4c136,39245,1732670739775 2024-11-27T01:26:20,016 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7d6f26a4c136:39245. 2024-11-27T01:26:20,016 INFO [RS:2;7d6f26a4c136:43859 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7d6f26a4c136:43859. 2024-11-27T01:26:20,016 DEBUG [RS:2;7d6f26a4c136:43859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:26:20,016 DEBUG [RS:1;7d6f26a4c136:39245 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T01:26:20,016 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ed6a1e752c9fdb4bc0ccc2dad778184, disabling compactions & flushes 2024-11-27T01:26:20,016 DEBUG [RS:2;7d6f26a4c136:43859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,016 DEBUG [RS:1;7d6f26a4c136:39245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,016 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:26:20,016 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,43859,1732670739814; all regions closed. 2024-11-27T01:26:20,016 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:26:20,016 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. after waiting 0 ms 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:26:20,016 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:26:20,016 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T01:26:20,017 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T01:26:20,017 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6ed6a1e752c9fdb4bc0ccc2dad778184=testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184.} 2024-11-27T01:26:20,017 DEBUG [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6ed6a1e752c9fdb4bc0ccc2dad778184 2024-11-27T01:26:20,017 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T01:26:20,017 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T01:26:20,017 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T01:26:20,017 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T01:26:20,017 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T01:26:20,018 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-27T01:26:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741834_1010 (size=175) 2024-11-27T01:26:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741834_1010 (size=175) 2024-11-27T01:26:20,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741834_1010 (size=175) 2024-11-27T01:26:20,023 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6ed6a1e752c9fdb4bc0ccc2dad778184/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-27T01:26:20,024 INFO [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:26:20,024 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ed6a1e752c9fdb4bc0ccc2dad778184: Waiting for close lock at 1732670780016Running coprocessor pre-close hooks at 1732670780016Disabling compacts and flushes for region at 1732670780016Disabling writes for close at 1732670780016Writing region close event to WAL at 1732670780017 (+1 ms)Running coprocessor post-close hooks at 1732670780023 (+6 ms)Closed at 1732670780023 2024-11-27T01:26:20,024 DEBUG [RS_CLOSE_REGION-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184. 2024-11-27T01:26:20,025 DEBUG [RS:2;7d6f26a4c136:43859 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C43859%2C1732670739814:(num 1732670741264) 2024-11-27T01:26:20,025 DEBUG [RS:2;7d6f26a4c136:43859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T01:26:20,025 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:26:20,025 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:26:20,026 INFO [RS:2;7d6f26a4c136:43859 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43859 2024-11-27T01:26:20,026 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:26:20,026 INFO [regionserver/7d6f26a4c136:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:26:20,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:26:20,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,43859,1732670739814 2024-11-27T01:26:20,028 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:26:20,029 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,43859,1732670739814] 2024-11-27T01:26:20,031 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,43859,1732670739814 already deleted, retry=false 2024-11-27T01:26:20,032 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,43859,1732670739814 expired; onlineServers=1 2024-11-27T01:26:20,041 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/info/d35f6128896049608139ac7b3cc0b50a is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732670755636.6ed6a1e752c9fdb4bc0ccc2dad778184./info:regioninfo/1732670758834/Put/seqid=0 2024-11-27T01:26:20,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741938_1118 (size=8243) 2024-11-27T01:26:20,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741938_1118 (size=8243) 2024-11-27T01:26:20,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741938_1118 (size=8243) 2024-11-27T01:26:20,050 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/info/d35f6128896049608139ac7b3cc0b50a 2024-11-27T01:26:20,070 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/ns/141ac0914f794d73a8d0833af5660fba is 43, key is default/ns:d/1732670741790/Put/seqid=0 2024-11-27T01:26:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741939_1119 (size=5153) 2024-11-27T01:26:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741939_1119 (size=5153) 2024-11-27T01:26:20,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741939_1119 (size=5153) 2024-11-27T01:26:20,077 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/ns/141ac0914f794d73a8d0833af5660fba 2024-11-27T01:26:20,093 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T01:26:20,093 INFO [regionserver/7d6f26a4c136:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T01:26:20,104 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/table/e74ff3b8de63452ba6b67ca6a9febf2e is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732670756077/Put/seqid=0 2024-11-27T01:26:20,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741940_1120 (size=5431) 2024-11-27T01:26:20,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741940_1120 (size=5431) 2024-11-27T01:26:20,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741940_1120 (size=5431) 2024-11-27T01:26:20,111 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/table/e74ff3b8de63452ba6b67ca6a9febf2e 2024-11-27T01:26:20,117 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/info/d35f6128896049608139ac7b3cc0b50a as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/info/d35f6128896049608139ac7b3cc0b50a 2024-11-27T01:26:20,122 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/info/d35f6128896049608139ac7b3cc0b50a, entries=18, sequenceid=21, filesize=8.0 K 2024-11-27T01:26:20,123 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/ns/141ac0914f794d73a8d0833af5660fba as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/ns/141ac0914f794d73a8d0833af5660fba 2024-11-27T01:26:20,127 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/ns/141ac0914f794d73a8d0833af5660fba, entries=2, sequenceid=21, filesize=5.0 K 2024-11-27T01:26:20,128 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/.tmp/table/e74ff3b8de63452ba6b67ca6a9febf2e as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/table/e74ff3b8de63452ba6b67ca6a9febf2e 2024-11-27T01:26:20,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:26:20,129 INFO [RS:2;7d6f26a4c136:43859 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:26:20,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43859-0x101570739b00003, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:26:20,129 INFO [RS:2;7d6f26a4c136:43859 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,43859,1732670739814; zookeeper connection closed. 2024-11-27T01:26:20,130 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c1cc31 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c1cc31 2024-11-27T01:26:20,134 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/table/e74ff3b8de63452ba6b67ca6a9febf2e, entries=2, sequenceid=21, filesize=5.3 K 2024-11-27T01:26:20,135 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=21, compaction requested=false 2024-11-27T01:26:20,140 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-27T01:26:20,140 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T01:26:20,140 INFO [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T01:26:20,141 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732670780017Running coprocessor pre-close hooks at 1732670780017Disabling compacts and flushes for region at 1732670780017Disabling writes for close at 1732670780017Obtaining lock to block concurrent updates at 1732670780018 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732670780018Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732670780018Flushing stores of hbase:meta,,1.1588230740 at 1732670780019 (+1 ms)Flushing 1588230740/info: creating writer at 1732670780019Flushing 1588230740/info: appending metadata at 1732670780041 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732670780041Flushing 1588230740/ns: creating writer at 1732670780055 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732670780069 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732670780070 (+1 ms)Flushing 1588230740/table: creating writer at 1732670780082 (+12 ms)Flushing 1588230740/table: appending metadata at 1732670780103 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732670780103Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b96b69e: reopening flushed file at 1732670780116 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5162fae4: reopening flushed file at 1732670780122 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b2c2dce: reopening flushed file at 1732670780128 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=21, compaction requested=false at 1732670780135 (+7 ms)Writing region close event to WAL at 1732670780136 (+1 ms)Running coprocessor post-close hooks at 1732670780140 (+4 ms)Closed at 1732670780140 2024-11-27T01:26:20,141 DEBUG [RS_CLOSE_META-regionserver/7d6f26a4c136:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T01:26:20,217 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(976): stopping server 7d6f26a4c136,39245,1732670739775; all regions closed. 2024-11-27T01:26:20,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741836_1012 (size=9487) 2024-11-27T01:26:20,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741836_1012 (size=9487) 2024-11-27T01:26:20,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741836_1012 (size=9487) 2024-11-27T01:26:20,222 DEBUG [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs 2024-11-27T01:26:20,222 INFO [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C39245%2C1732670739775.meta:.meta(num 1732670741608) 2024-11-27T01:26:20,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741833_1009 (size=3621) 2024-11-27T01:26:20,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741833_1009 (size=3621) 2024-11-27T01:26:20,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741833_1009 (size=3621) 2024-11-27T01:26:20,228 DEBUG [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/oldWALs 2024-11-27T01:26:20,228 INFO [RS:1;7d6f26a4c136:39245 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d6f26a4c136%2C39245%2C1732670739775:(num 1732670741263) 2024-11-27T01:26:20,228 DEBUG [RS:1;7d6f26a4c136:39245 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T01:26:20,228 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T01:26:20,228 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:26:20,228 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.ChoreService(370): Chore service for: regionserver/7d6f26a4c136:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T01:26:20,228 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:26:20,228 INFO [regionserver/7d6f26a4c136:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:26:20,229 INFO [RS:1;7d6f26a4c136:39245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39245 2024-11-27T01:26:20,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T01:26:20,231 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:26:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d6f26a4c136,39245,1732670739775 2024-11-27T01:26:20,232 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d6f26a4c136,39245,1732670739775] 2024-11-27T01:26:20,233 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d6f26a4c136,39245,1732670739775 already deleted, retry=false 2024-11-27T01:26:20,233 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d6f26a4c136,39245,1732670739775 expired; onlineServers=0 2024-11-27T01:26:20,233 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7d6f26a4c136,37429,1732670738986' ***** 2024-11-27T01:26:20,233 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T01:26:20,233 INFO [M:0;7d6f26a4c136:37429 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T01:26:20,234 INFO [M:0;7d6f26a4c136:37429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T01:26:20,234 DEBUG [M:0;7d6f26a4c136:37429 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T01:26:20,234 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T01:26:20,234 DEBUG [M:0;7d6f26a4c136:37429 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T01:26:20,234 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670740890 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.large.0-1732670740890,5,FailOnTimeoutGroup] 2024-11-27T01:26:20,234 DEBUG [master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670740891 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d6f26a4c136:0:becomeActiveMaster-HFileCleaner.small.0-1732670740891,5,FailOnTimeoutGroup] 2024-11-27T01:26:20,234 INFO [M:0;7d6f26a4c136:37429 {}] hbase.ChoreService(370): Chore service for: master/7d6f26a4c136:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T01:26:20,234 INFO [M:0;7d6f26a4c136:37429 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T01:26:20,234 DEBUG [M:0;7d6f26a4c136:37429 {}] master.HMaster(1795): Stopping service threads 2024-11-27T01:26:20,234 INFO [M:0;7d6f26a4c136:37429 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T01:26:20,234 INFO [M:0;7d6f26a4c136:37429 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T01:26:20,235 INFO [M:0;7d6f26a4c136:37429 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T01:26:20,235 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T01:26:20,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T01:26:20,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T01:26:20,237 DEBUG [M:0;7d6f26a4c136:37429 {}] zookeeper.ZKUtil(347): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T01:26:20,237 WARN [M:0;7d6f26a4c136:37429 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T01:26:20,237 INFO [M:0;7d6f26a4c136:37429 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/.lastflushedseqids 2024-11-27T01:26:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741941_1121 (size=138) 2024-11-27T01:26:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741941_1121 (size=138) 2024-11-27T01:26:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741941_1121 (size=138) 2024-11-27T01:26:20,250 INFO [M:0;7d6f26a4c136:37429 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T01:26:20,251 INFO [M:0;7d6f26a4c136:37429 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T01:26:20,251 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T01:26:20,251 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:26:20,251 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:26:20,251 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T01:26:20,251 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:26:20,251 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-27T01:26:20,267 DEBUG [M:0;7d6f26a4c136:37429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/768b1ca4d3fc40a3a9673f87261aeb84 is 82, key is hbase:meta,,1/info:regioninfo/1732670741717/Put/seqid=0 2024-11-27T01:26:20,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741942_1122 (size=5672) 2024-11-27T01:26:20,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741942_1122 (size=5672) 2024-11-27T01:26:20,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741942_1122 (size=5672) 2024-11-27T01:26:20,273 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/768b1ca4d3fc40a3a9673f87261aeb84 2024-11-27T01:26:20,296 DEBUG [M:0;7d6f26a4c136:37429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b536b2741bd443f8824252729ce2268 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732670756084/Put/seqid=0 2024-11-27T01:26:20,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741943_1123 (size=7755) 2024-11-27T01:26:20,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741943_1123 (size=7755) 2024-11-27T01:26:20,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741943_1123 (size=7755) 2024-11-27T01:26:20,303 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b536b2741bd443f8824252729ce2268 2024-11-27T01:26:20,308 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6b536b2741bd443f8824252729ce2268 2024-11-27T01:26:20,323 DEBUG [M:0;7d6f26a4c136:37429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/059bd8229d304761b5404e4d59ee0da4 is 69, key is 7d6f26a4c136,39245,1732670739775/rs:state/1732670741014/Put/seqid=0 2024-11-27T01:26:20,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741944_1124 (size=5445) 2024-11-27T01:26:20,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741944_1124 (size=5445) 2024-11-27T01:26:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741944_1124 (size=5445) 2024-11-27T01:26:20,330 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/059bd8229d304761b5404e4d59ee0da4 2024-11-27T01:26:20,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:26:20,332 INFO [RS:1;7d6f26a4c136:39245 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:26:20,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39245-0x101570739b00002, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:26:20,332 INFO [RS:1;7d6f26a4c136:39245 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d6f26a4c136,39245,1732670739775; zookeeper connection closed. 2024-11-27T01:26:20,333 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f0b8d4e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f0b8d4e 2024-11-27T01:26:20,333 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T01:26:20,335 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 059bd8229d304761b5404e4d59ee0da4 2024-11-27T01:26:20,336 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/768b1ca4d3fc40a3a9673f87261aeb84 as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/768b1ca4d3fc40a3a9673f87261aeb84 2024-11-27T01:26:20,342 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/768b1ca4d3fc40a3a9673f87261aeb84, entries=8, sequenceid=168, filesize=5.5 K 2024-11-27T01:26:20,343 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6b536b2741bd443f8824252729ce2268 as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6b536b2741bd443f8824252729ce2268 2024-11-27T01:26:20,348 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6b536b2741bd443f8824252729ce2268 2024-11-27T01:26:20,348 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6b536b2741bd443f8824252729ce2268, entries=17, sequenceid=168, filesize=7.6 K 2024-11-27T01:26:20,349 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/059bd8229d304761b5404e4d59ee0da4 as hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/059bd8229d304761b5404e4d59ee0da4 2024-11-27T01:26:20,354 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 059bd8229d304761b5404e4d59ee0da4 2024-11-27T01:26:20,354 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36155/user/jenkins/test-data/fa41af66-0111-a9d6-1d3e-60cf5ceedad1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/059bd8229d304761b5404e4d59ee0da4, entries=3, sequenceid=168, filesize=5.3 K 2024-11-27T01:26:20,356 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false 2024-11-27T01:26:20,357 INFO [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T01:26:20,357 DEBUG [M:0;7d6f26a4c136:37429 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732670780251Disabling compacts and flushes for region at 1732670780251Disabling writes for close at 1732670780251Obtaining lock to block concurrent updates at 1732670780251Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732670780251Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732670780251Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732670780252 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732670780252Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732670780266 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732670780266Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732670780277 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732670780295 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732670780295Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732670780308 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732670780322 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732670780322Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@357cbd5: reopening flushed file at 1732670780336 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27704f00: reopening flushed file at 1732670780342 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d99867f: reopening flushed file at 1732670780349 (+7 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=168, compaction requested=false at 1732670780356 (+7 ms)Writing region close event to WAL at 1732670780357 (+1 ms)Closed at 1732670780357 2024-11-27T01:26:20,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44005 is added to blk_1073741830_1006 (size=83606) 2024-11-27T01:26:20,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39261 is added to blk_1073741830_1006 (size=83606) 2024-11-27T01:26:20,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45665 is added to blk_1073741830_1006 (size=83606) 2024-11-27T01:26:20,361 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T01:26:20,361 INFO [M:0;7d6f26a4c136:37429 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T01:26:20,361 INFO [M:0;7d6f26a4c136:37429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37429 2024-11-27T01:26:20,361 INFO [M:0;7d6f26a4c136:37429 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T01:26:20,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:26:20,463 INFO [M:0;7d6f26a4c136:37429 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T01:26:20,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37429-0x101570739b00000, quorum=127.0.0.1:58346, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T01:26:20,470 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670760019 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670760019 (inode 16655) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732670759143/wal.1732670760019 (inode 16655) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:26:20,471 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732670750849/wal.1732670750987 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:20,471 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732670775187/wal.1732670775475 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:20,473 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670769538 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670769538 (inode 16678) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732670760211/wal.1732670769538 (inode 16678) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:26:20,474 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732670743862/wal.1732670743957 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:20,476 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670770414 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670770414 (inode 16704) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732670769799/wal.1732670770414 (inode 16704) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:26:20,477 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732670770617/wal.1732670770699 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T01:26:20,479 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal.1732670743679 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal.1732670743679 (inode 16485) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732670742839/wal.1732670743679 (inode 16485) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:26:20,482 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal.1732670742523 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal.1732670742523 (inode 16462) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732670742044/wal.1732670742523 (inode 16462) Holder DFSClient_NONMAPREDUCE_-674852374_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-27T01:26:20,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@773b4a40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:26:20,488 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286fa8e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:26:20,488 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:26:20,489 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b0de5e3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:26:20,489 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78fad685{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,STOPPED} 2024-11-27T01:26:20,491 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:26:20,491 WARN [BP-879573077-172.17.0.2-1732670735999 heartbeating to localhost/127.0.0.1:36155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:26:20,491 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:26:20,491 WARN [BP-879573077-172.17.0.2-1732670735999 heartbeating to localhost/127.0.0.1:36155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-879573077-172.17.0.2-1732670735999 (Datanode Uuid 246cac20-e37a-4e3e-9564-168acfc5ca14) service to localhost/127.0.0.1:36155 2024-11-27T01:26:20,493 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data5/current/BP-879573077-172.17.0.2-1732670735999 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:26:20,493 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data6/current/BP-879573077-172.17.0.2-1732670735999 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:26:20,494 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:26:20,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75d96791{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:26:20,496 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60aef58b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:26:20,496 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:26:20,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14fd625f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:26:20,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78a777d4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,STOPPED} 2024-11-27T01:26:20,497 WARN [BP-879573077-172.17.0.2-1732670735999 heartbeating to localhost/127.0.0.1:36155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:26:20,497 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:26:20,497 WARN [BP-879573077-172.17.0.2-1732670735999 heartbeating to localhost/127.0.0.1:36155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-879573077-172.17.0.2-1732670735999 (Datanode Uuid 1c2e6c23-04a2-455c-8937-48d461463669) service to localhost/127.0.0.1:36155 2024-11-27T01:26:20,498 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:26:20,498 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data3/current/BP-879573077-172.17.0.2-1732670735999 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:26:20,498 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data4/current/BP-879573077-172.17.0.2-1732670735999 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:26:20,499 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:26:20,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d480c14{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T01:26:20,501 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8a1e1ee{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:26:20,501 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:26:20,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ace2437{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:26:20,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49d86915{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,STOPPED} 2024-11-27T01:26:20,503 WARN [BP-879573077-172.17.0.2-1732670735999 heartbeating to localhost/127.0.0.1:36155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T01:26:20,503 WARN [BP-879573077-172.17.0.2-1732670735999 heartbeating to localhost/127.0.0.1:36155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-879573077-172.17.0.2-1732670735999 (Datanode Uuid 971d4915-7e58-4935-b45e-daabb70295fa) service to localhost/127.0.0.1:36155 2024-11-27T01:26:20,503 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T01:26:20,503 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T01:26:20,503 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data1/current/BP-879573077-172.17.0.2-1732670735999 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:26:20,503 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/cluster_1d718db4-4db1-dcff-01fd-ffa02e0543e9/data/data2/current/BP-879573077-172.17.0.2-1732670735999 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T01:26:20,504 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T01:26:20,510 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14f32243{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T01:26:20,511 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71a2a7f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T01:26:20,511 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T01:26:20,511 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@120a4c3b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T01:26:20,511 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a6ae9ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/17e75772-5297-5dc1-aacd-4a1ce3f0b51f/hadoop.log.dir/,STOPPED} 2024-11-27T01:26:20,520 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T01:26:20,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down