2024-12-08 01:19:09,120 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-08 01:19:09,131 main DEBUG Took 0.009760 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 01:19:09,132 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 01:19:09,132 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 01:19:09,133 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 01:19:09,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,142 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 01:19:09,152 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,155 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,156 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,157 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,158 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,159 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,160 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,160 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,160 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,161 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,161 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 01:19:09,162 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,162 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 01:19:09,163 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 01:19:09,165 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 01:19:09,166 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 01:19:09,167 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 01:19:09,168 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 01:19:09,168 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 01:19:09,175 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 01:19:09,177 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 01:19:09,178 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 01:19:09,179 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 01:19:09,179 main DEBUG createAppenders(={Console}) 2024-12-08 01:19:09,180 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-08 01:19:09,180 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-08 01:19:09,180 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-08 01:19:09,181 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 01:19:09,181 main DEBUG OutputStream closed 2024-12-08 01:19:09,181 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 01:19:09,181 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 01:19:09,181 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-08 01:19:09,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 01:19:09,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 01:19:09,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 01:19:09,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 01:19:09,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 01:19:09,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 01:19:09,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 01:19:09,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 01:19:09,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 01:19:09,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 01:19:09,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 01:19:09,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 01:19:09,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 01:19:09,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 01:19:09,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 01:19:09,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 01:19:09,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 01:19:09,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 01:19:09,256 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 01:19:09,256 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-08 01:19:09,257 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 01:19:09,257 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-08T01:19:09,461 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e 2024-12-08 01:19:09,463 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 01:19:09,464 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T01:19:09,472 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-08T01:19:09,478 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-08T01:19:09,499 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T01:19:09,542 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T01:19:09,543 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T01:19:09,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T01:19:09,567 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9, deleteOnExit=true 2024-12-08T01:19:09,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T01:19:09,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/test.cache.data in system properties and HBase conf 2024-12-08T01:19:09,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T01:19:09,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir in system properties and HBase conf 2024-12-08T01:19:09,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T01:19:09,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T01:19:09,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T01:19:09,637 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T01:19:09,727 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T01:19:09,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T01:19:09,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T01:19:09,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T01:19:09,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T01:19:09,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T01:19:09,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T01:19:09,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T01:19:09,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T01:19:09,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T01:19:09,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/nfs.dump.dir in system properties and HBase conf 2024-12-08T01:19:09,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/java.io.tmpdir in system properties and HBase conf 2024-12-08T01:19:09,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T01:19:09,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T01:19:09,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T01:19:10,836 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T01:19:10,902 INFO [Time-limited test {}] log.Log(170): Logging initialized @2382ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T01:19:10,964 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:19:11,021 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:19:11,045 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:19:11,045 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:19:11,047 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T01:19:11,058 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:19:11,061 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:19:11,062 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:19:11,223 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/java.io.tmpdir/jetty-localhost-45291-hadoop-hdfs-3_4_1-tests_jar-_-any-8672687049703466035/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T01:19:11,229 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:45291} 2024-12-08T01:19:11,230 INFO [Time-limited test {}] server.Server(415): Started @2710ms 2024-12-08T01:19:11,856 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:19:11,864 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:19:11,865 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:19:11,865 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:19:11,865 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T01:19:11,866 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:19:11,867 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:19:11,964 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/java.io.tmpdir/jetty-localhost-37359-hadoop-hdfs-3_4_1-tests_jar-_-any-1579207586992188687/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:11,965 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:37359} 2024-12-08T01:19:11,965 INFO [Time-limited test {}] server.Server(415): Started @3446ms 2024-12-08T01:19:12,005 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:19:12,100 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:19:12,107 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:19:12,110 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:19:12,110 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:19:12,111 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T01:19:12,113 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:19:12,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:19:12,223 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/java.io.tmpdir/jetty-localhost-42937-hadoop-hdfs-3_4_1-tests_jar-_-any-4241063146403781636/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:12,224 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:42937} 2024-12-08T01:19:12,224 INFO [Time-limited test {}] server.Server(415): Started @3704ms 2024-12-08T01:19:12,226 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:19:12,259 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T01:19:12,263 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T01:19:12,265 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T01:19:12,265 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T01:19:12,265 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T01:19:12,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,AVAILABLE} 2024-12-08T01:19:12,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T01:19:12,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/java.io.tmpdir/jetty-localhost-40111-hadoop-hdfs-3_4_1-tests_jar-_-any-12805470941712812290/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:12,371 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:40111} 2024-12-08T01:19:12,372 INFO [Time-limited test {}] server.Server(415): Started @3852ms 2024-12-08T01:19:12,373 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T01:19:13,760 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data5/current/BP-1381908131-172.17.0.2-1733620750211/current, will proceed with Du for space computation calculation, 2024-12-08T01:19:13,760 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data2/current/BP-1381908131-172.17.0.2-1733620750211/current, will proceed with Du for space computation calculation, 2024-12-08T01:19:13,760 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data3/current/BP-1381908131-172.17.0.2-1733620750211/current, will proceed with Du for space computation calculation, 2024-12-08T01:19:13,760 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data6/current/BP-1381908131-172.17.0.2-1733620750211/current, will proceed with Du for space computation calculation, 2024-12-08T01:19:13,760 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data4/current/BP-1381908131-172.17.0.2-1733620750211/current, will proceed with Du for space computation calculation, 2024-12-08T01:19:13,760 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data1/current/BP-1381908131-172.17.0.2-1733620750211/current, will proceed with Du for space computation calculation, 2024-12-08T01:19:13,800 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:19:13,801 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:19:13,801 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T01:19:13,844 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd29b29c35c58043c with lease ID 0x83be871631e94662: Processing first storage report for DS-39851136-a6e8-487a-a185-d8070836f45c from datanode DatanodeRegistration(127.0.0.1:41533, datanodeUuid=9d496315-77bf-4f0f-91fe-bfe62675722e, infoPort=38357, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211) 2024-12-08T01:19:13,846 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd29b29c35c58043c with lease ID 0x83be871631e94662: from storage DS-39851136-a6e8-487a-a185-d8070836f45c node DatanodeRegistration(127.0.0.1:41533, datanodeUuid=9d496315-77bf-4f0f-91fe-bfe62675722e, infoPort=38357, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-08T01:19:13,846 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x860dc3b012da6462 with lease ID 0x83be871631e94664: Processing first storage report for DS-c4fd417a-751e-4030-9465-13c368067924 from datanode DatanodeRegistration(127.0.0.1:39867, datanodeUuid=51479df6-55ff-431e-b610-6e5a5f4eaf1f, infoPort=39561, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211) 2024-12-08T01:19:13,846 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x860dc3b012da6462 with lease ID 0x83be871631e94664: from storage DS-c4fd417a-751e-4030-9465-13c368067924 node DatanodeRegistration(127.0.0.1:39867, datanodeUuid=51479df6-55ff-431e-b610-6e5a5f4eaf1f, infoPort=39561, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:19:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c699152b2ef293c with lease ID 0x83be871631e94663: Processing first storage report for DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3 from datanode DatanodeRegistration(127.0.0.1:39611, datanodeUuid=3b43399b-4a4d-46eb-aaeb-19c0ebf380e8, infoPort=33735, infoSecurePort=0, ipcPort=36935, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211) 2024-12-08T01:19:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c699152b2ef293c with lease ID 0x83be871631e94663: from storage DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3 node DatanodeRegistration(127.0.0.1:39611, datanodeUuid=3b43399b-4a4d-46eb-aaeb-19c0ebf380e8, infoPort=33735, infoSecurePort=0, ipcPort=36935, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:19:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd29b29c35c58043c with lease ID 0x83be871631e94662: Processing first storage report for DS-05615c44-7738-4999-93b5-f5df8cec51b1 from datanode DatanodeRegistration(127.0.0.1:41533, datanodeUuid=9d496315-77bf-4f0f-91fe-bfe62675722e, infoPort=38357, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211) 2024-12-08T01:19:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd29b29c35c58043c with lease ID 0x83be871631e94662: from storage DS-05615c44-7738-4999-93b5-f5df8cec51b1 node DatanodeRegistration(127.0.0.1:41533, datanodeUuid=9d496315-77bf-4f0f-91fe-bfe62675722e, infoPort=38357, infoSecurePort=0, ipcPort=43593, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:19:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x860dc3b012da6462 with lease ID 0x83be871631e94664: Processing first storage report for DS-2e6cd23a-e9c3-43ee-bba4-bdf0112f70fb from datanode DatanodeRegistration(127.0.0.1:39867, datanodeUuid=51479df6-55ff-431e-b610-6e5a5f4eaf1f, infoPort=39561, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211) 2024-12-08T01:19:13,848 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x860dc3b012da6462 with lease ID 0x83be871631e94664: from storage DS-2e6cd23a-e9c3-43ee-bba4-bdf0112f70fb node DatanodeRegistration(127.0.0.1:39867, datanodeUuid=51479df6-55ff-431e-b610-6e5a5f4eaf1f, infoPort=39561, infoSecurePort=0, ipcPort=34557, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-08T01:19:13,848 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c699152b2ef293c with lease ID 0x83be871631e94663: Processing first storage report for DS-38ec135e-673e-4be5-9485-4cbcacbbf05d from datanode DatanodeRegistration(127.0.0.1:39611, datanodeUuid=3b43399b-4a4d-46eb-aaeb-19c0ebf380e8, infoPort=33735, infoSecurePort=0, ipcPort=36935, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211) 2024-12-08T01:19:13,848 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c699152b2ef293c with lease ID 0x83be871631e94663: from storage DS-38ec135e-673e-4be5-9485-4cbcacbbf05d node DatanodeRegistration(127.0.0.1:39611, datanodeUuid=3b43399b-4a4d-46eb-aaeb-19c0ebf380e8, infoPort=33735, infoSecurePort=0, ipcPort=36935, storageInfo=lv=-57;cid=testClusterID;nsid=1812763930;c=1733620750211), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T01:19:13,874 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e 2024-12-08T01:19:13,947 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/zookeeper_0, clientPort=54213, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T01:19:13,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54213 2024-12-08T01:19:13,966 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:13,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:14,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:19:14,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:19:14,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741825_1001 (size=7) 2024-12-08T01:19:14,575 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae with version=8 2024-12-08T01:19:14,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/hbase-staging 2024-12-08T01:19:14,826 INFO [Time-limited test {}] client.ConnectionUtils(128): master/96982154c400:0 server-side Connection retries=45 2024-12-08T01:19:14,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:14,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:14,838 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:19:14,838 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:14,838 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:19:14,949 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T01:19:15,000 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T01:19:15,008 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T01:19:15,011 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:19:15,034 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 55663 (auto-detected) 2024-12-08T01:19:15,035 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T01:19:15,053 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39725 2024-12-08T01:19:15,071 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39725 connecting to ZooKeeper ensemble=127.0.0.1:54213 2024-12-08T01:19:15,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397250x0, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:19:15,191 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39725-0x10003016cce0000 connected 2024-12-08T01:19:15,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,290 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:15,306 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae, hbase.cluster.distributed=false 2024-12-08T01:19:15,326 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:19:15,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39725 2024-12-08T01:19:15,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39725 2024-12-08T01:19:15,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39725 2024-12-08T01:19:15,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39725 2024-12-08T01:19:15,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39725 2024-12-08T01:19:15,423 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:19:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,425 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:19:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,425 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:19:15,428 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:19:15,430 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:19:15,431 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35801 2024-12-08T01:19:15,433 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35801 connecting to ZooKeeper ensemble=127.0.0.1:54213 2024-12-08T01:19:15,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,437 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358010x0, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:19:15,622 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35801-0x10003016cce0001 connected 2024-12-08T01:19:15,622 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:15,629 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:19:15,639 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:19:15,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:19:15,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:19:15,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35801 2024-12-08T01:19:15,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35801 2024-12-08T01:19:15,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35801 2024-12-08T01:19:15,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35801 2024-12-08T01:19:15,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35801 2024-12-08T01:19:15,662 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:19:15,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,663 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:19:15,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:19:15,663 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:19:15,664 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:19:15,664 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37871 2024-12-08T01:19:15,666 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37871 connecting to ZooKeeper ensemble=127.0.0.1:54213 2024-12-08T01:19:15,667 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378710x0, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:19:15,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378710x0, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:15,717 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37871-0x10003016cce0002 connected 2024-12-08T01:19:15,718 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:19:15,719 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:19:15,720 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:19:15,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:19:15,723 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37871 2024-12-08T01:19:15,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37871 2024-12-08T01:19:15,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37871 2024-12-08T01:19:15,725 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37871 2024-12-08T01:19:15,725 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37871 2024-12-08T01:19:15,742 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/96982154c400:0 server-side Connection retries=45 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T01:19:15,743 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T01:19:15,744 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43235 2024-12-08T01:19:15,745 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43235 connecting to ZooKeeper ensemble=127.0.0.1:54213 2024-12-08T01:19:15,746 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,748 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:15,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432350x0, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T01:19:15,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:432350x0, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:15,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43235-0x10003016cce0003 connected 2024-12-08T01:19:15,805 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T01:19:15,807 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T01:19:15,808 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T01:19:15,812 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T01:19:15,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43235 2024-12-08T01:19:15,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43235 2024-12-08T01:19:15,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43235 2024-12-08T01:19:15,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43235 2024-12-08T01:19:15,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43235 2024-12-08T01:19:15,828 DEBUG [M:0;96982154c400:39725 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;96982154c400:39725 2024-12-08T01:19:15,829 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/96982154c400,39725,1733620754685 2024-12-08T01:19:15,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,855 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/96982154c400,39725,1733620754685 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:15,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:15,885 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T01:19:15,886 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/96982154c400,39725,1733620754685 from backup master directory 2024-12-08T01:19:15,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/96982154c400,39725,1733620754685 2024-12-08T01:19:15,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T01:19:15,895 WARN [master/96982154c400:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:19:15,896 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=96982154c400,39725,1733620754685 2024-12-08T01:19:15,899 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T01:19:15,901 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T01:19:15,949 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/hbase.id] with ID: 48c17904-c0cd-482c-afe5-f922c94d85d2 2024-12-08T01:19:15,949 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/.tmp/hbase.id 2024-12-08T01:19:15,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:19:15,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:19:15,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741826_1002 (size=42) 2024-12-08T01:19:15,964 DEBUG [master/96982154c400:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/.tmp/hbase.id]:[hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/hbase.id] 2024-12-08T01:19:16,004 INFO [master/96982154c400:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T01:19:16,008 INFO [master/96982154c400:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T01:19:16,024 INFO [master/96982154c400:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-08T01:19:16,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:19:16,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:19:16,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741827_1003 (size=196) 2024-12-08T01:19:16,065 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:19:16,066 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T01:19:16,071 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:16,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:19:16,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:19:16,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741828_1004 (size=1189) 2024-12-08T01:19:16,117 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store 2024-12-08T01:19:16,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:19:16,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:19:16,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741829_1005 (size=34) 2024-12-08T01:19:16,140 INFO [master/96982154c400:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T01:19:16,142 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:16,144 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T01:19:16,144 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:16,144 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:16,145 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T01:19:16,145 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:16,145 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:16,146 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733620756143Disabling compacts and flushes for region at 1733620756143Disabling writes for close at 1733620756145 (+2 ms)Writing region close event to WAL at 1733620756145Closed at 1733620756145 2024-12-08T01:19:16,148 WARN [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/.initializing 2024-12-08T01:19:16,148 DEBUG [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/WALs/96982154c400,39725,1733620754685 2024-12-08T01:19:16,156 INFO [master/96982154c400:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:16,170 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C39725%2C1733620754685, suffix=, logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/WALs/96982154c400,39725,1733620754685, archiveDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/oldWALs, maxLogs=10 2024-12-08T01:19:16,205 DEBUG [master/96982154c400:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/WALs/96982154c400,39725,1733620754685/96982154c400%2C39725%2C1733620754685.1733620756176, exclude list is [], retry=0 2024-12-08T01:19:16,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:16,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:16,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:16,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:16,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T01:19:16,254 DEBUG [master/96982154c400:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-08T01:19:16,306 INFO [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/WALs/96982154c400,39725,1733620754685/96982154c400%2C39725%2C1733620754685.1733620756176 2024-12-08T01:19:16,307 DEBUG [master/96982154c400:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:16,307 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:16,307 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:16,310 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,311 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,360 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T01:19:16,363 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:16,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T01:19:16,368 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,369 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:16,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T01:19:16,372 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:16,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T01:19:16,376 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,377 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:16,378 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,382 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,384 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,389 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,389 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,393 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:19:16,396 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T01:19:16,400 DEBUG [master/96982154c400:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:16,401 INFO [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73156171, jitterRate=0.09011189639568329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:19:16,407 DEBUG [master/96982154c400:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733620756321Initializing all the Stores at 1733620756323 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620756323Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620756324 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620756324Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620756324Cleaning up temporary data from old regions at 1733620756390 (+66 ms)Region opened successfully at 1733620756407 (+17 ms) 2024-12-08T01:19:16,408 INFO [master/96982154c400:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T01:19:16,439 DEBUG [master/96982154c400:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79224913, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:19:16,464 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T01:19:16,473 INFO [master/96982154c400:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T01:19:16,473 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T01:19:16,475 INFO [master/96982154c400:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T01:19:16,476 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T01:19:16,481 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-08T01:19:16,481 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T01:19:16,501 INFO [master/96982154c400:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T01:19:16,508 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T01:19:16,547 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T01:19:16,551 INFO [master/96982154c400:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T01:19:16,554 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T01:19:16,567 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T01:19:16,570 INFO [master/96982154c400:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T01:19:16,575 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T01:19:16,588 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T01:19:16,591 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T01:19:16,600 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T01:19:16,622 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T01:19:16,631 DEBUG [master/96982154c400:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,645 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=96982154c400,39725,1733620754685, sessionid=0x10003016cce0000, setting cluster-up flag (Was=false) 2024-12-08T01:19:16,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,706 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T01:19:16,710 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=96982154c400,39725,1733620754685 2024-12-08T01:19:16,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:16,768 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T01:19:16,771 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=96982154c400,39725,1733620754685 2024-12-08T01:19:16,779 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T01:19:16,818 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(746): ClusterId : 48c17904-c0cd-482c-afe5-f922c94d85d2 2024-12-08T01:19:16,818 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(746): ClusterId : 48c17904-c0cd-482c-afe5-f922c94d85d2 2024-12-08T01:19:16,818 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(746): ClusterId : 48c17904-c0cd-482c-afe5-f922c94d85d2 2024-12-08T01:19:16,821 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:19:16,821 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:19:16,821 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T01:19:16,833 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:19:16,833 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:19:16,833 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T01:19:16,833 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:19:16,833 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:19:16,833 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T01:19:16,843 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:19:16,843 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:19:16,843 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T01:19:16,844 DEBUG [RS:1;96982154c400:37871 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f722f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:19:16,844 DEBUG [RS:2;96982154c400:43235 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@524b0708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:19:16,844 DEBUG [RS:0;96982154c400:35801 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656cb0c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=96982154c400/172.17.0.2:0 2024-12-08T01:19:16,843 DEBUG [master/96982154c400:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T01:19:16,855 INFO [master/96982154c400:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T01:19:16,863 DEBUG [RS:0;96982154c400:35801 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;96982154c400:35801 2024-12-08T01:19:16,863 DEBUG [RS:2;96982154c400:43235 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;96982154c400:43235 2024-12-08T01:19:16,863 DEBUG [RS:1;96982154c400:37871 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;96982154c400:37871 2024-12-08T01:19:16,863 INFO [master/96982154c400:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T01:19:16,866 INFO [RS:1;96982154c400:37871 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:19:16,866 INFO [RS:2;96982154c400:43235 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:19:16,866 INFO [RS:0;96982154c400:35801 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T01:19:16,867 INFO [RS:0;96982154c400:35801 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:19:16,867 INFO [RS:1;96982154c400:37871 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:19:16,867 INFO [RS:2;96982154c400:43235 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T01:19:16,867 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:19:16,867 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:19:16,867 DEBUG [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T01:19:16,869 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,39725,1733620754685 with port=37871, startcode=1733620755662 2024-12-08T01:19:16,869 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,39725,1733620754685 with port=43235, startcode=1733620755742 2024-12-08T01:19:16,869 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(2659): reportForDuty to master=96982154c400,39725,1733620754685 with port=35801, startcode=1733620755391 2024-12-08T01:19:16,869 DEBUG [master/96982154c400:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 96982154c400,39725,1733620754685 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T01:19:16,875 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:19:16,875 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:19:16,876 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:19:16,876 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/96982154c400:0, corePoolSize=5, maxPoolSize=5 2024-12-08T01:19:16,876 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/96982154c400:0, corePoolSize=10, maxPoolSize=10 2024-12-08T01:19:16,876 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:16,876 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:19:16,876 DEBUG [master/96982154c400:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:16,880 DEBUG [RS:0;96982154c400:35801 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:19:16,880 DEBUG [RS:2;96982154c400:43235 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:19:16,880 DEBUG [RS:1;96982154c400:37871 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T01:19:16,883 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733620786883 2024-12-08T01:19:16,884 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:19:16,885 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T01:19:16,885 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T01:19:16,886 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T01:19:16,890 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T01:19:16,890 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T01:19:16,890 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,890 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T01:19:16,891 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T01:19:16,891 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T01:19:16,891 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:16,895 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T01:19:16,896 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T01:19:16,897 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T01:19:16,900 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T01:19:16,900 INFO [master/96982154c400:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T01:19:16,904 DEBUG [master/96982154c400:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620756902,5,FailOnTimeoutGroup] 2024-12-08T01:19:16,904 DEBUG [master/96982154c400:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620756904,5,FailOnTimeoutGroup] 2024-12-08T01:19:16,905 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:16,905 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T01:19:16,942 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:16,943 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:16,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:19:16,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:19:16,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741831_1007 (size=1321) 2024-12-08T01:19:16,951 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T01:19:16,951 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae 2024-12-08T01:19:16,956 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41461, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:19:16,956 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37663, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:19:16,956 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57193, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T01:19:16,962 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,37871,1733620755662 2024-12-08T01:19:16,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741832_1008 (size=32) 2024-12-08T01:19:16,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741832_1008 (size=32) 2024-12-08T01:19:16,965 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] master.ServerManager(517): Registering regionserver=96982154c400,37871,1733620755662 2024-12-08T01:19:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741832_1008 (size=32) 2024-12-08T01:19:16,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:16,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:19:16,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:19:16,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:16,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:19:16,975 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,43235,1733620755742 2024-12-08T01:19:16,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:19:16,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,975 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] master.ServerManager(517): Registering regionserver=96982154c400,43235,1733620755742 2024-12-08T01:19:16,976 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:16,977 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:19:16,979 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae 2024-12-08T01:19:16,979 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35931 2024-12-08T01:19:16,979 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:19:16,979 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:19:16,979 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:16,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:19:16,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 96982154c400,35801,1733620755391 2024-12-08T01:19:16,981 DEBUG [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae 2024-12-08T01:19:16,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] master.ServerManager(517): Registering regionserver=96982154c400,35801,1733620755391 2024-12-08T01:19:16,981 DEBUG [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35931 2024-12-08T01:19:16,981 DEBUG [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:19:16,985 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae 2024-12-08T01:19:16,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:19:16,985 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35931 2024-12-08T01:19:16,985 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T01:19:16,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:16,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:16,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:19:16,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740 2024-12-08T01:19:16,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740 2024-12-08T01:19:16,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:19:16,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:19:16,992 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:19:16,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:19:16,998 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:16,999 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70701422, jitterRate=0.05353328585624695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:19:17,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733620756967Initializing all the Stores at 1733620756968 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620756969 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620756969Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620756969Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620756969Cleaning up temporary data from old regions at 1733620756991 (+22 ms)Region opened successfully at 1733620757001 (+10 ms) 2024-12-08T01:19:17,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:19:17,002 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:19:17,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:19:17,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:19:17,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:19:17,003 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:17,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620757001Disabling compacts and flushes for region at 1733620757001Disabling writes for close at 1733620757002 (+1 ms)Writing region close event to WAL at 1733620757003 (+1 ms)Closed at 1733620757003 2024-12-08T01:19:17,005 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:19:17,006 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T01:19:17,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T01:19:17,018 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:19:17,021 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T01:19:17,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:19:17,042 DEBUG [RS:1;96982154c400:37871 {}] zookeeper.ZKUtil(111): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,37871,1733620755662 2024-12-08T01:19:17,042 DEBUG [RS:2;96982154c400:43235 {}] zookeeper.ZKUtil(111): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,43235,1733620755742 2024-12-08T01:19:17,042 WARN [RS:1;96982154c400:37871 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:19:17,042 WARN [RS:2;96982154c400:43235 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:19:17,042 INFO [RS:1;96982154c400:37871 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:17,042 INFO [RS:2;96982154c400:43235 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:17,043 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662 2024-12-08T01:19:17,043 DEBUG [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,43235,1733620755742 2024-12-08T01:19:17,043 DEBUG [RS:0;96982154c400:35801 {}] zookeeper.ZKUtil(111): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/96982154c400,35801,1733620755391 2024-12-08T01:19:17,043 WARN [RS:0;96982154c400:35801 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T01:19:17,043 INFO [RS:0;96982154c400:35801 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:17,043 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391 2024-12-08T01:19:17,045 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,43235,1733620755742] 2024-12-08T01:19:17,045 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,37871,1733620755662] 2024-12-08T01:19:17,045 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [96982154c400,35801,1733620755391] 2024-12-08T01:19:17,069 INFO [RS:1;96982154c400:37871 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:19:17,069 INFO [RS:0;96982154c400:35801 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:19:17,069 INFO [RS:2;96982154c400:43235 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T01:19:17,084 INFO [RS:0;96982154c400:35801 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:19:17,084 INFO [RS:2;96982154c400:43235 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:19:17,084 INFO [RS:1;96982154c400:37871 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T01:19:17,091 INFO [RS:0;96982154c400:35801 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:19:17,091 INFO [RS:2;96982154c400:43235 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:19:17,091 INFO [RS:1;96982154c400:37871 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:19:17,091 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,091 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,091 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,092 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:19:17,092 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:19:17,092 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T01:19:17,097 INFO [RS:0;96982154c400:35801 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:19:17,097 INFO [RS:1;96982154c400:37871 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:19:17,097 INFO [RS:2;96982154c400:43235 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T01:19:17,098 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,098 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,098 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,099 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:19:17,099 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,099 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/96982154c400:0, corePoolSize=2, maxPoolSize=2 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:0;96982154c400:35801 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:19:17,100 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,100 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,101 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/96982154c400:0, corePoolSize=1, maxPoolSize=1 2024-12-08T01:19:17,101 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:19:17,101 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:19:17,101 DEBUG [RS:2;96982154c400:43235 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:19:17,101 DEBUG [RS:1;96982154c400:37871 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/96982154c400:0, corePoolSize=3, maxPoolSize=3 2024-12-08T01:19:17,102 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,35801,1733620755391-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:19:17,102 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37871,1733620755662-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,103 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,43235,1733620755742-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:19:17,119 INFO [RS:0;96982154c400:35801 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:19:17,121 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,35801,1733620755391-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,122 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,122 INFO [RS:0;96982154c400:35801 {}] regionserver.Replication(171): 96982154c400,35801,1733620755391 started 2024-12-08T01:19:17,124 INFO [RS:1;96982154c400:37871 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:19:17,124 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,37871,1733620755662-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,125 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,125 INFO [RS:1;96982154c400:37871 {}] regionserver.Replication(171): 96982154c400,37871,1733620755662 started 2024-12-08T01:19:17,126 INFO [RS:2;96982154c400:43235 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T01:19:17,126 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,43235,1733620755742-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,126 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,127 INFO [RS:2;96982154c400:43235 {}] regionserver.Replication(171): 96982154c400,43235,1733620755742 started 2024-12-08T01:19:17,144 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,144 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,35801,1733620755391, RpcServer on 96982154c400/172.17.0.2:35801, sessionid=0x10003016cce0001 2024-12-08T01:19:17,145 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,145 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,37871,1733620755662, RpcServer on 96982154c400/172.17.0.2:37871, sessionid=0x10003016cce0002 2024-12-08T01:19:17,145 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:19:17,145 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:19:17,145 DEBUG [RS:1;96982154c400:37871 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,37871,1733620755662 2024-12-08T01:19:17,145 DEBUG [RS:0;96982154c400:35801 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,35801,1733620755391 2024-12-08T01:19:17,145 DEBUG [RS:1;96982154c400:37871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,37871,1733620755662' 2024-12-08T01:19:17,146 DEBUG [RS:0;96982154c400:35801 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,35801,1733620755391' 2024-12-08T01:19:17,146 DEBUG [RS:1;96982154c400:37871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:19:17,146 DEBUG [RS:0;96982154c400:35801 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:19:17,146 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,146 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(1482): Serving as 96982154c400,43235,1733620755742, RpcServer on 96982154c400/172.17.0.2:43235, sessionid=0x10003016cce0003 2024-12-08T01:19:17,146 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T01:19:17,147 DEBUG [RS:2;96982154c400:43235 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 96982154c400,43235,1733620755742 2024-12-08T01:19:17,147 DEBUG [RS:1;96982154c400:37871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:19:17,147 DEBUG [RS:2;96982154c400:43235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,43235,1733620755742' 2024-12-08T01:19:17,147 DEBUG [RS:0;96982154c400:35801 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:19:17,147 DEBUG [RS:2;96982154c400:43235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T01:19:17,147 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:19:17,147 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:19:17,147 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:19:17,147 DEBUG [RS:2;96982154c400:43235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T01:19:17,147 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:19:17,147 DEBUG [RS:1;96982154c400:37871 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,37871,1733620755662 2024-12-08T01:19:17,147 DEBUG [RS:0;96982154c400:35801 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,35801,1733620755391 2024-12-08T01:19:17,148 DEBUG [RS:0;96982154c400:35801 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,35801,1733620755391' 2024-12-08T01:19:17,148 DEBUG [RS:1;96982154c400:37871 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,37871,1733620755662' 2024-12-08T01:19:17,148 DEBUG [RS:0;96982154c400:35801 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:19:17,148 DEBUG [RS:1;96982154c400:37871 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:19:17,148 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T01:19:17,148 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T01:19:17,148 DEBUG [RS:2;96982154c400:43235 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 96982154c400,43235,1733620755742 2024-12-08T01:19:17,148 DEBUG [RS:2;96982154c400:43235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '96982154c400,43235,1733620755742' 2024-12-08T01:19:17,148 DEBUG [RS:2;96982154c400:43235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T01:19:17,148 DEBUG [RS:0;96982154c400:35801 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:19:17,148 DEBUG [RS:1;96982154c400:37871 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:19:17,149 DEBUG [RS:0;96982154c400:35801 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:19:17,149 DEBUG [RS:1;96982154c400:37871 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:19:17,149 DEBUG [RS:2;96982154c400:43235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T01:19:17,149 INFO [RS:0;96982154c400:35801 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:19:17,149 INFO [RS:1;96982154c400:37871 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:19:17,149 INFO [RS:1;96982154c400:37871 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:19:17,149 INFO [RS:0;96982154c400:35801 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:19:17,149 DEBUG [RS:2;96982154c400:43235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T01:19:17,149 INFO [RS:2;96982154c400:43235 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T01:19:17,149 INFO [RS:2;96982154c400:43235 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T01:19:17,172 WARN [96982154c400:39725 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-08T01:19:17,254 INFO [RS:0;96982154c400:35801 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:17,254 INFO [RS:2;96982154c400:43235 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:17,254 INFO [RS:1;96982154c400:37871 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:17,257 INFO [RS:0;96982154c400:35801 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C35801%2C1733620755391, suffix=, logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391, archiveDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs, maxLogs=32 2024-12-08T01:19:17,257 INFO [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C37871%2C1733620755662, suffix=, logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662, archiveDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs, maxLogs=32 2024-12-08T01:19:17,257 INFO [RS:2;96982154c400:43235 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C43235%2C1733620755742, suffix=, logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,43235,1733620755742, archiveDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs, maxLogs=32 2024-12-08T01:19:17,271 DEBUG [RS:2;96982154c400:43235 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,43235,1733620755742/96982154c400%2C43235%2C1733620755742.1733620757260, exclude list is [], retry=0 2024-12-08T01:19:17,273 DEBUG [RS:0;96982154c400:35801 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391/96982154c400%2C35801%2C1733620755391.1733620757260, exclude list is [], retry=0 2024-12-08T01:19:17,273 DEBUG [RS:1;96982154c400:37871 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662/96982154c400%2C37871%2C1733620755662.1733620757260, exclude list is [], retry=0 2024-12-08T01:19:17,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:17,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:17,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:17,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:17,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:17,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:17,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:17,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:17,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:17,313 INFO [RS:0;96982154c400:35801 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391/96982154c400%2C35801%2C1733620755391.1733620757260 2024-12-08T01:19:17,313 INFO [RS:2;96982154c400:43235 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,43235,1733620755742/96982154c400%2C43235%2C1733620755742.1733620757260 2024-12-08T01:19:17,315 DEBUG [RS:0;96982154c400:35801 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:17,315 DEBUG [RS:2;96982154c400:43235 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:17,318 INFO [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662/96982154c400%2C37871%2C1733620755662.1733620757260 2024-12-08T01:19:17,319 DEBUG [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:17,425 DEBUG [96982154c400:39725 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-08T01:19:17,435 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:19:17,442 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:19:17,443 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:19:17,443 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-08T01:19:17,443 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:19:17,443 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:19:17,443 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-08T01:19:17,443 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:19:17,443 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:19:17,443 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-08T01:19:17,443 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:19:17,450 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:17,455 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,35801,1733620755391, state=OPENING 2024-12-08T01:19:17,494 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T01:19:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:17,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:17,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,508 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,510 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:19:17,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,35801,1733620755391}] 2024-12-08T01:19:17,691 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:19:17,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:19:17,703 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T01:19:17,704 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:17,704 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T01:19:17,708 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C35801%2C1733620755391.meta, suffix=.meta, logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391, archiveDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs, maxLogs=32 2024-12-08T01:19:17,721 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, exclude list is [], retry=0 2024-12-08T01:19:17,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:17,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:17,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:17,752 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta 2024-12-08T01:19:17,753 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:17,753 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:17,754 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T01:19:17,756 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T01:19:17,761 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T01:19:17,764 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T01:19:17,765 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:17,765 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T01:19:17,765 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T01:19:17,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:19:17,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:19:17,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:17,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:17,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:19:17,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:19:17,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:17,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:17,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:19:17,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:19:17,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:17,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:17,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:19:17,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:19:17,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:17,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:17,779 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:19:17,780 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740 2024-12-08T01:19:17,783 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740 2024-12-08T01:19:17,785 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:19:17,786 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:19:17,786 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:19:17,789 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:19:17,791 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59921998, jitterRate=-0.10709264874458313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:19:17,791 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T01:19:17,792 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733620757765Writing region info on filesystem at 1733620757765Initializing all the Stores at 1733620757767 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620757767Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620757768 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620757768Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620757768Cleaning up temporary data from old regions at 1733620757786 (+18 ms)Running coprocessor post-open hooks at 1733620757791 (+5 ms)Region opened successfully at 1733620757792 (+1 ms) 2024-12-08T01:19:17,798 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733620757681 2024-12-08T01:19:17,808 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T01:19:17,809 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T01:19:17,811 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:17,813 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,35801,1733620755391, state=OPEN 2024-12-08T01:19:17,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:17,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:17,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:17,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:17,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,821 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:17,821 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=96982154c400,35801,1733620755391 2024-12-08T01:19:17,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T01:19:17,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,35801,1733620755391 in 308 msec 2024-12-08T01:19:17,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T01:19:17,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 817 msec 2024-12-08T01:19:17,836 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T01:19:17,836 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T01:19:17,855 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:17,856 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1] 2024-12-08T01:19:17,875 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:19:17,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:19:17,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0890 sec 2024-12-08T01:19:17,897 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733620757897, completionTime=-1 2024-12-08T01:19:17,900 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-08T01:19:17,900 DEBUG [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T01:19:17,924 INFO [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-08T01:19:17,924 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733620817924 2024-12-08T01:19:17,924 INFO [master/96982154c400:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733620877924 2024-12-08T01:19:17,924 INFO [master/96982154c400:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-08T01:19:17,925 DEBUG [master/96982154c400:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-08T01:19:17,931 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,39725,1733620754685-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,932 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,39725,1733620754685-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,932 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,39725,1733620754685-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,933 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-96982154c400:39725, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,934 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,934 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:17,940 DEBUG [master/96982154c400:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T01:19:17,986 INFO [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.089sec 2024-12-08T01:19:17,987 INFO [master/96982154c400:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T01:19:17,988 INFO [master/96982154c400:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T01:19:17,989 INFO [master/96982154c400:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T01:19:17,989 INFO [master/96982154c400:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T01:19:17,990 INFO [master/96982154c400:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T01:19:17,990 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,39725,1733620754685-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T01:19:17,991 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,39725,1733620754685-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T01:19:17,998 DEBUG [master/96982154c400:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T01:19:17,999 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T01:19:17,999 INFO [master/96982154c400:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=96982154c400,39725,1733620754685-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:18,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a5f0dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:19:18,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 96982154c400,39725,-1 for getting cluster id 2024-12-08T01:19:18,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T01:19:18,038 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '48c17904-c0cd-482c-afe5-f922c94d85d2' 2024-12-08T01:19:18,040 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T01:19:18,040 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "48c17904-c0cd-482c-afe5-f922c94d85d2" 2024-12-08T01:19:18,041 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32049f67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:19:18,041 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [96982154c400,39725,-1] 2024-12-08T01:19:18,043 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T01:19:18,045 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:18,046 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T01:19:18,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f754b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T01:19:18,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:18,055 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1] 2024-12-08T01:19:18,055 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:19:18,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:19:18,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=96982154c400,39725,1733620754685 2024-12-08T01:19:18,082 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35931/hbase 2024-12-08T01:19:18,095 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=369, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=10400 2024-12-08T01:19:18,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:18,123 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:18,124 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:18,129 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98513462, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-98513462, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:18,149 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98513462/hregion-98513462.1733620758131, exclude list is [], retry=0 2024-12-08T01:19:18,154 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:18,155 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:18,155 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:18,181 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98513462/hregion-98513462.1733620758131 2024-12-08T01:19:18,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:18,182 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f0f099467125bf799ff4e2d4e8dfbab9, NAME => 'testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741838_1014 (size=64) 2024-12-08T01:19:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741838_1014 (size=64) 2024-12-08T01:19:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741838_1014 (size=64) 2024-12-08T01:19:18,196 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:18,198 INFO [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,201 INFO [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0f099467125bf799ff4e2d4e8dfbab9 columnFamilyName a 2024-12-08T01:19:18,201 DEBUG [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:18,202 INFO [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] regionserver.HStore(327): Store=f0f099467125bf799ff4e2d4e8dfbab9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:18,202 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,205 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,205 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,207 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,211 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:18,212 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f0f099467125bf799ff4e2d4e8dfbab9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74216283, jitterRate=0.10590879619121552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f0f099467125bf799ff4e2d4e8dfbab9: Writing region info on filesystem at 1733620758196Initializing all the Stores at 1733620758198 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620758198Cleaning up temporary data from old regions at 1733620758205 (+7 ms)Region opened successfully at 1733620758212 (+7 ms) 2024-12-08T01:19:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f0f099467125bf799ff4e2d4e8dfbab9, disabling compactions & flushes 2024-12-08T01:19:18,213 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9. 2024-12-08T01:19:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9. 2024-12-08T01:19:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9. after waiting 0 ms 2024-12-08T01:19:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9. 2024-12-08T01:19:18,213 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9. 2024-12-08T01:19:18,213 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f0f099467125bf799ff4e2d4e8dfbab9: Waiting for close lock at 1733620758213Disabling compacts and flushes for region at 1733620758213Disabling writes for close at 1733620758213Writing region close event to WAL at 1733620758213Closed at 1733620758213 2024-12-08T01:19:18,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741837_1013 (size=175) 2024-12-08T01:19:18,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741837_1013 (size=175) 2024-12-08T01:19:18,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741837_1013 (size=175) 2024-12-08T01:19:18,224 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:18,224 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98513462:(num 1733620758131) 2024-12-08T01:19:18,226 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:19:18,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741839_1015 (size=469) 2024-12-08T01:19:18,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741839_1015 (size=469) 2024-12-08T01:19:18,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741839_1015 (size=469) 2024-12-08T01:19:18,273 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:19:18,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741840_1016 (size=322) 2024-12-08T01:19:18,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741840_1016 (size=322) 2024-12-08T01:19:18,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741840_1016 (size=322) 2024-12-08T01:19:18,328 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1, size=469 (469bytes) 2024-12-08T01:19:18,329 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-08T01:19:18,329 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-08T01:19:18,329 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1 2024-12-08T01:19:18,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1 after 4ms 2024-12-08T01:19:18,363 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:18,364 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1 took 37ms 2024-12-08T01:19:18,370 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1 so closing down 2024-12-08T01:19:18,370 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:18,372 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-08T01:19:18,397 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000001-wal-1.temp 2024-12-08T01:19:18,398 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741841_1017 (size=469) 2024-12-08T01:19:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741841_1017 (size=469) 2024-12-08T01:19:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741841_1017 (size=469) 2024-12-08T01:19:18,409 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:18,411 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002 2024-12-08T01:19:18,415 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-08T01:19:18,415 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1, journal: Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1, size=469 (469bytes) at 1733620758328Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1 so closing down at 1733620758370 (+42 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000001-wal-1.temp at 1733620758397 (+27 ms)3 split writer threads finished at 1733620758398 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733620758409 (+11 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002 at 1733620758411 (+2 ms)Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733620758415 (+4 ms) 2024-12-08T01:19:18,428 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2, size=322 (322bytes) 2024-12-08T01:19:18,428 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2 2024-12-08T01:19:18,428 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2 after 0ms 2024-12-08T01:19:18,451 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:18,451 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2 took 24ms 2024-12-08T01:19:18,454 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2 so closing down 2024-12-08T01:19:18,454 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:18,457 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-08T01:19:18,479 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002-wal-2.temp 2024-12-08T01:19:18,480 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:18,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741842_1018 (size=322) 2024-12-08T01:19:18,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741842_1018 (size=322) 2024-12-08T01:19:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741842_1018 (size=322) 2024-12-08T01:19:18,491 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:18,516 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:18,519 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-08T01:19:18,521 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-08T01:19:18,521 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2, journal: Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2, size=322 (322bytes) at 1733620758428Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2 so closing down at 1733620758454 (+26 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002-wal-2.temp at 1733620758479 (+25 ms)3 split writer threads finished at 1733620758480 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733620758491 (+11 ms)Processed 1 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733620758521 (+30 ms) 2024-12-08T01:19:18,521 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:18,524 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:18,537 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal.1733620758525, exclude list is [], retry=0 2024-12-08T01:19:18,541 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:18,542 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:18,542 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:18,567 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal.1733620758525 2024-12-08T01:19:18,567 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:18,567 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f0f099467125bf799ff4e2d4e8dfbab9, NAME => 'testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:18,567 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:18,567 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,567 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,570 INFO [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,571 INFO [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f0f099467125bf799ff4e2d4e8dfbab9 columnFamilyName a 2024-12-08T01:19:18,571 DEBUG [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:18,572 INFO [StoreOpener-f0f099467125bf799ff4e2d4e8dfbab9-1 {}] regionserver.HStore(327): Store=f0f099467125bf799ff4e2d4e8dfbab9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:18,572 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,574 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,577 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,578 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002 2024-12-08T01:19:18,605 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:18,610 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002 2024-12-08T01:19:18,613 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f0f099467125bf799ff4e2d4e8dfbab9 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-08T01:19:18,656 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/.tmp/a/190a18fef0744c9e9dd03bf47120819d is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733620758224/Put/seqid=0 2024-12-08T01:19:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741844_1020 (size=5170) 2024-12-08T01:19:18,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741844_1020 (size=5170) 2024-12-08T01:19:18,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741844_1020 (size=5170) 2024-12-08T01:19:18,669 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/.tmp/a/190a18fef0744c9e9dd03bf47120819d 2024-12-08T01:19:18,709 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/.tmp/a/190a18fef0744c9e9dd03bf47120819d as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/a/190a18fef0744c9e9dd03bf47120819d 2024-12-08T01:19:18,719 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/a/190a18fef0744c9e9dd03bf47120819d, entries=2, sequenceid=2, filesize=5.0 K 2024-12-08T01:19:18,724 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f0f099467125bf799ff4e2d4e8dfbab9 in 110ms, sequenceid=2, compaction requested=false; wal=null 2024-12-08T01:19:18,726 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/0000000000000000002 2024-12-08T01:19:18,727 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,727 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,730 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f0f099467125bf799ff4e2d4e8dfbab9 2024-12-08T01:19:18,733 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/f0f099467125bf799ff4e2d4e8dfbab9/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-08T01:19:18,735 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f0f099467125bf799ff4e2d4e8dfbab9; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59651126, jitterRate=-0.11112895607948303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:18,735 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f0f099467125bf799ff4e2d4e8dfbab9: Writing region info on filesystem at 1733620758568Initializing all the Stores at 1733620758569 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620758569Obtaining lock to block concurrent updates at 1733620758613 (+44 ms)Preparing flush snapshotting stores in f0f099467125bf799ff4e2d4e8dfbab9 at 1733620758613Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733620758616 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733620758119.f0f099467125bf799ff4e2d4e8dfbab9. at 1733620758616Flushing f0f099467125bf799ff4e2d4e8dfbab9/a: creating writer at 1733620758617 (+1 ms)Flushing f0f099467125bf799ff4e2d4e8dfbab9/a: appending metadata at 1733620758647 (+30 ms)Flushing f0f099467125bf799ff4e2d4e8dfbab9/a: closing flushed file at 1733620758650 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364a9d8a: reopening flushed file at 1733620758707 (+57 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f0f099467125bf799ff4e2d4e8dfbab9 in 110ms, sequenceid=2, compaction requested=false; wal=null at 1733620758724 (+17 ms)Cleaning up temporary data from old regions at 1733620758727 (+3 ms)Region opened successfully at 1733620758735 (+8 ms) 2024-12-08T01:19:18,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=380 (was 369) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:53580 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:53734 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:42684 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50124 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35931/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50252 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=10379 (was 10400) 2024-12-08T01:19:18,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=10378 2024-12-08T01:19:18,784 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:18,786 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:18,787 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:18,790 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54480387, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-54480387, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:18,802 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54480387/hregion-54480387.1733620758791, exclude list is [], retry=0 2024-12-08T01:19:18,806 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:18,806 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:18,806 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:18,831 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54480387/hregion-54480387.1733620758791 2024-12-08T01:19:18,832 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:18,832 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5e1ce77acda63d4eb376e74fe6d258c1, NAME => 'testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:18,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741846_1022 (size=64) 2024-12-08T01:19:18,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741846_1022 (size=64) 2024-12-08T01:19:18,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741846_1022 (size=64) 2024-12-08T01:19:18,846 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:18,848 INFO [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,850 INFO [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e1ce77acda63d4eb376e74fe6d258c1 columnFamilyName a 2024-12-08T01:19:18,850 DEBUG [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:18,851 INFO [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] regionserver.HStore(327): Store=5e1ce77acda63d4eb376e74fe6d258c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:18,851 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,852 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,853 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,853 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,856 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:18,859 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:18,860 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e1ce77acda63d4eb376e74fe6d258c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63631892, jitterRate=-0.05181092023849487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:18,860 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e1ce77acda63d4eb376e74fe6d258c1: Writing region info on filesystem at 1733620758846Initializing all the Stores at 1733620758848 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620758848Cleaning up temporary data from old regions at 1733620758854 (+6 ms)Region opened successfully at 1733620758860 (+6 ms) 2024-12-08T01:19:18,860 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5e1ce77acda63d4eb376e74fe6d258c1, disabling compactions & flushes 2024-12-08T01:19:18,860 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1. 2024-12-08T01:19:18,860 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1. 2024-12-08T01:19:18,861 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1. after waiting 0 ms 2024-12-08T01:19:18,861 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1. 2024-12-08T01:19:18,861 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1. 2024-12-08T01:19:18,861 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5e1ce77acda63d4eb376e74fe6d258c1: Waiting for close lock at 1733620758860Disabling compacts and flushes for region at 1733620758860Disabling writes for close at 1733620758861 (+1 ms)Writing region close event to WAL at 1733620758861Closed at 1733620758861 2024-12-08T01:19:18,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741845_1021 (size=175) 2024-12-08T01:19:18,865 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-54480387/hregion-54480387.1733620758791 not finished, retry = 0 2024-12-08T01:19:18,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741845_1021 (size=175) 2024-12-08T01:19:18,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741845_1021 (size=175) 2024-12-08T01:19:18,969 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:18,969 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54480387:(num 1733620758791) 2024-12-08T01:19:18,970 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:19:19,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741847_1023 (size=469) 2024-12-08T01:19:19,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741847_1023 (size=469) 2024-12-08T01:19:19,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741847_1023 (size=469) 2024-12-08T01:19:19,005 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-08T01:19:19,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741848_1024 (size=322) 2024-12-08T01:19:19,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741848_1024 (size=322) 2024-12-08T01:19:19,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741848_1024 (size=322) 2024-12-08T01:19:19,054 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2, size=322 (322bytes) 2024-12-08T01:19:19,054 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2 2024-12-08T01:19:19,055 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2 after 1ms 2024-12-08T01:19:19,078 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:19,079 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2 took 25ms 2024-12-08T01:19:19,081 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2 so closing down 2024-12-08T01:19:19,081 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:19,083 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-08T01:19:19,107 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002-wal-2.temp 2024-12-08T01:19:19,107 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:19,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741849_1025 (size=322) 2024-12-08T01:19:19,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741849_1025 (size=322) 2024-12-08T01:19:19,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741849_1025 (size=322) 2024-12-08T01:19:19,115 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:19,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 2024-12-08T01:19:19,117 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-08T01:19:19,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2, journal: Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2, size=322 (322bytes) at 1733620759054Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2 so closing down at 1733620759081 (+27 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002-wal-2.temp at 1733620759107 (+26 ms)3 split writer threads finished at 1733620759107Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733620759115 (+8 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 at 1733620759117 (+2 ms)Processed 1 edits across 1 Regions in 38 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733620759118 (+1 ms) 2024-12-08T01:19:19,131 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1, size=469 (469bytes) 2024-12-08T01:19:19,131 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1 2024-12-08T01:19:19,132 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1 after 1ms 2024-12-08T01:19:19,155 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:19,155 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1 took 24ms 2024-12-08T01:19:19,158 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1 so closing down 2024-12-08T01:19:19,158 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:19,160 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-08T01:19:19,182 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000001-wal-1.temp 2024-12-08T01:19:19,183 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741850_1026 (size=469) 2024-12-08T01:19:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741850_1026 (size=469) 2024-12-08T01:19:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741850_1026 (size=469) 2024-12-08T01:19:19,191 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:19,218 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:19,221 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002, length=322 2024-12-08T01:19:19,223 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 2024-12-08T01:19:19,223 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-08T01:19:19,224 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1, journal: Splitting hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1, size=469 (469bytes) at 1733620759131Finishing writing output for hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1 so closing down at 1733620759158 (+27 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000001-wal-1.temp at 1733620759182 (+24 ms)3 split writer threads finished at 1733620759183 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733620759191 (+8 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 at 1733620759223 (+32 ms)Processed 2 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733620759223 2024-12-08T01:19:19,224 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:19,226 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:19,243 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal.1733620759227, exclude list is [], retry=0 2024-12-08T01:19:19,246 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:19,246 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:19,247 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:19,271 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal.1733620759227 2024-12-08T01:19:19,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:19,271 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e1ce77acda63d4eb376e74fe6d258c1, NAME => 'testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:19,271 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:19,271 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,271 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,274 INFO [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,275 INFO [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e1ce77acda63d4eb376e74fe6d258c1 columnFamilyName a 2024-12-08T01:19:19,276 DEBUG [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:19,276 INFO [StoreOpener-5e1ce77acda63d4eb376e74fe6d258c1-1 {}] regionserver.HStore(327): Store=5e1ce77acda63d4eb376e74fe6d258c1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:19,277 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,277 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,280 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,281 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 2024-12-08T01:19:19,308 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:19,310 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 2024-12-08T01:19:19,310 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5e1ce77acda63d4eb376e74fe6d258c1 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-08T01:19:19,325 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/.tmp/a/83467bdcb4ce4cc2a7fba779381cb708 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733620758969/Put/seqid=0 2024-12-08T01:19:19,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741852_1028 (size=5170) 2024-12-08T01:19:19,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741852_1028 (size=5170) 2024-12-08T01:19:19,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741852_1028 (size=5170) 2024-12-08T01:19:19,335 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/.tmp/a/83467bdcb4ce4cc2a7fba779381cb708 2024-12-08T01:19:19,345 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/.tmp/a/83467bdcb4ce4cc2a7fba779381cb708 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/a/83467bdcb4ce4cc2a7fba779381cb708 2024-12-08T01:19:19,353 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/a/83467bdcb4ce4cc2a7fba779381cb708, entries=2, sequenceid=2, filesize=5.0 K 2024-12-08T01:19:19,354 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5e1ce77acda63d4eb376e74fe6d258c1 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-12-08T01:19:19,355 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/0000000000000000002 2024-12-08T01:19:19,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,358 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5e1ce77acda63d4eb376e74fe6d258c1 2024-12-08T01:19:19,361 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/5e1ce77acda63d4eb376e74fe6d258c1/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-08T01:19:19,363 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5e1ce77acda63d4eb376e74fe6d258c1; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62049969, jitterRate=-0.07538340985774994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:19,363 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5e1ce77acda63d4eb376e74fe6d258c1: Writing region info on filesystem at 1733620759272Initializing all the Stores at 1733620759273 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620759273Obtaining lock to block concurrent updates at 1733620759310 (+37 ms)Preparing flush snapshotting stores in 5e1ce77acda63d4eb376e74fe6d258c1 at 1733620759310Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733620759311 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733620758784.5e1ce77acda63d4eb376e74fe6d258c1. at 1733620759311Flushing 5e1ce77acda63d4eb376e74fe6d258c1/a: creating writer at 1733620759311Flushing 5e1ce77acda63d4eb376e74fe6d258c1/a: appending metadata at 1733620759324 (+13 ms)Flushing 5e1ce77acda63d4eb376e74fe6d258c1/a: closing flushed file at 1733620759324Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7797afc8: reopening flushed file at 1733620759343 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5e1ce77acda63d4eb376e74fe6d258c1 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1733620759354 (+11 ms)Cleaning up temporary data from old regions at 1733620759355 (+1 ms)Region opened successfully at 1733620759363 (+8 ms) 2024-12-08T01:19:19,378 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:42756 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50124 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:42766 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:53580 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50316 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:53806 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=10359 (was 10378) 2024-12-08T01:19:19,390 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=10358 2024-12-08T01:19:19,405 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:19,407 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:19,408 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:19,411 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-87376598, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-87376598, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:19,423 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-87376598/hregion-87376598.1733620759412, exclude list is [], retry=0 2024-12-08T01:19:19,426 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:19,426 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:19,426 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:19,449 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-87376598/hregion-87376598.1733620759412 2024-12-08T01:19:19,451 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:19,451 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 075fde603788b946bb2e6204d28ea0a9, NAME => 'testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741854_1030 (size=64) 2024-12-08T01:19:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741854_1030 (size=64) 2024-12-08T01:19:19,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741854_1030 (size=64) 2024-12-08T01:19:19,462 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:19,464 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,466 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075fde603788b946bb2e6204d28ea0a9 columnFamilyName a 2024-12-08T01:19:19,466 DEBUG [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:19,467 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(327): Store=075fde603788b946bb2e6204d28ea0a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:19,467 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,468 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075fde603788b946bb2e6204d28ea0a9 columnFamilyName b 2024-12-08T01:19:19,468 DEBUG [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:19,469 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(327): Store=075fde603788b946bb2e6204d28ea0a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:19,469 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,471 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075fde603788b946bb2e6204d28ea0a9 columnFamilyName c 2024-12-08T01:19:19,471 DEBUG [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:19,471 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(327): Store=075fde603788b946bb2e6204d28ea0a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:19,471 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,472 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,474 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,474 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:19,476 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:19,478 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:19,479 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 075fde603788b946bb2e6204d28ea0a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70722931, jitterRate=0.05385379493236542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 075fde603788b946bb2e6204d28ea0a9: Writing region info on filesystem at 1733620759463Initializing all the Stores at 1733620759464 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620759464Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620759464Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620759464Cleaning up temporary data from old regions at 1733620759474 (+10 ms)Region opened successfully at 1733620759479 (+5 ms) 2024-12-08T01:19:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 075fde603788b946bb2e6204d28ea0a9, disabling compactions & flushes 2024-12-08T01:19:19,479 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. after waiting 0 ms 2024-12-08T01:19:19,479 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:19,480 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:19,480 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 075fde603788b946bb2e6204d28ea0a9: Waiting for close lock at 1733620759479Disabling compacts and flushes for region at 1733620759479Disabling writes for close at 1733620759479Writing region close event to WAL at 1733620759480 (+1 ms)Closed at 1733620759480 2024-12-08T01:19:19,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741853_1029 (size=175) 2024-12-08T01:19:19,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741853_1029 (size=175) 2024-12-08T01:19:19,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741853_1029 (size=175) 2024-12-08T01:19:19,487 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:19,487 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-87376598:(num 1733620759412) 2024-12-08T01:19:19,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:19,489 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:19,503 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490, exclude list is [], retry=0 2024-12-08T01:19:19,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:19,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:19,506 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:19,531 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 2024-12-08T01:19:19,532 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:19,709 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490, size=0 (0bytes) 2024-12-08T01:19:19,709 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 might be still open, length is 0 2024-12-08T01:19:19,709 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 2024-12-08T01:19:19,710 WARN [IPC Server handler 2 on default port 35931 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-08T01:19:19,710 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 after 1ms 2024-12-08T01:19:19,834 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:53850 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53850 dst: /127.0.0.1:41533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41533 remote=/127.0.0.1:53850]. Total timeout mills is 60000, 59847 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:19,835 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50364 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39611:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50364 dst: /127.0.0.1:39611 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:19,835 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:42802 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42802 dst: /127.0.0.1:39867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:19,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741855_1032 (size=497775) 2024-12-08T01:19:19,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741855_1032 (size=497775) 2024-12-08T01:19:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741855_1032 (size=497775) 2024-12-08T01:19:23,278 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:19:23,322 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T01:19:23,711 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 after 4002ms 2024-12-08T01:19:23,742 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:23,744 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 took 4035ms 2024-12-08T01:19:23,748 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733620759490.temp 2024-12-08T01:19:23,784 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000000001-wal.1733620759490.temp 2024-12-08T01:19:23,832 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490; continuing. 2024-12-08T01:19:23,832 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 so closing down 2024-12-08T01:19:23,832 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:23,832 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741856_1033 (size=497783) 2024-12-08T01:19:23,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741856_1033 (size=497783) 2024-12-08T01:19:23,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741856_1033 (size=497783) 2024-12-08T01:19:23,835 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000000001-wal.1733620759490.temp (wrote 3002 edits, skipped 0 edits in 39 ms) 2024-12-08T01:19:23,837 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000000001-wal.1733620759490.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000003002 2024-12-08T01:19:23,838 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 93 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:19:23,838 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490, journal: Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490, size=0 (0bytes) at 1733620759709Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000000001-wal.1733620759490.temp at 1733620763784 (+4075 ms)Split 1024 edits, skipped 0 edits. at 1733620763793 (+9 ms)Split 2048 edits, skipped 0 edits. at 1733620763814 (+21 ms)Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 so closing down at 1733620763832 (+18 ms)3 split writer threads finished at 1733620763832Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000000001-wal.1733620759490.temp (wrote 3002 edits, skipped 0 edits in 39 ms) at 1733620763835 (+3 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000000001-wal.1733620759490.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000003002 at 1733620763837 (+2 ms)Processed 3002 edits across 1 Regions in 93 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490, size=0, length=0, corrupted=false, cancelled=false at 1733620763838 (+1 ms) 2024-12-08T01:19:23,840 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620759490 2024-12-08T01:19:23,841 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000003002 2024-12-08T01:19:23,842 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:23,844 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:23,856 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620763844, exclude list is [], retry=0 2024-12-08T01:19:23,860 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:23,860 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:23,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:23,882 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620763844 2024-12-08T01:19:23,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:23,883 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:23,885 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:23,886 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075fde603788b946bb2e6204d28ea0a9 columnFamilyName a 2024-12-08T01:19:23,886 DEBUG [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:23,887 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(327): Store=075fde603788b946bb2e6204d28ea0a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:23,887 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:23,888 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075fde603788b946bb2e6204d28ea0a9 columnFamilyName b 2024-12-08T01:19:23,888 DEBUG [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:23,888 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(327): Store=075fde603788b946bb2e6204d28ea0a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:23,889 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:23,889 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 075fde603788b946bb2e6204d28ea0a9 columnFamilyName c 2024-12-08T01:19:23,889 DEBUG [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:23,890 INFO [StoreOpener-075fde603788b946bb2e6204d28ea0a9-1 {}] regionserver.HStore(327): Store=075fde603788b946bb2e6204d28ea0a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:23,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:23,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:23,893 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:23,894 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000003002 2024-12-08T01:19:23,918 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:23,946 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:19:24,256 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 075fde603788b946bb2e6204d28ea0a9 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-08T01:19:24,288 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/a/ab4919a1ffac4e67b51e8d94bd55d5c8 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733620759538/Put/seqid=0 2024-12-08T01:19:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741858_1035 (size=50463) 2024-12-08T01:19:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741858_1035 (size=50463) 2024-12-08T01:19:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741858_1035 (size=50463) 2024-12-08T01:19:24,295 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/a/ab4919a1ffac4e67b51e8d94bd55d5c8 2024-12-08T01:19:24,303 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/a/ab4919a1ffac4e67b51e8d94bd55d5c8 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/a/ab4919a1ffac4e67b51e8d94bd55d5c8 2024-12-08T01:19:24,310 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/a/ab4919a1ffac4e67b51e8d94bd55d5c8, entries=754, sequenceid=754, filesize=49.3 K 2024-12-08T01:19:24,310 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 075fde603788b946bb2e6204d28ea0a9 in 54ms, sequenceid=754, compaction requested=false; wal=null 2024-12-08T01:19:24,332 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:19:24,333 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 075fde603788b946bb2e6204d28ea0a9 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-08T01:19:24,342 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/a/24ebd2a2268f4fd2b5ed02405051594c is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733620759570/Put/seqid=0 2024-12-08T01:19:24,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741859_1036 (size=20072) 2024-12-08T01:19:24,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741859_1036 (size=20072) 2024-12-08T01:19:24,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741859_1036 (size=20072) 2024-12-08T01:19:24,352 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/a/24ebd2a2268f4fd2b5ed02405051594c 2024-12-08T01:19:24,376 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/b/b0691074ae0140a09f74955dbafd08de is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733620759591/Put/seqid=0 2024-12-08T01:19:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741860_1037 (size=35835) 2024-12-08T01:19:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741860_1037 (size=35835) 2024-12-08T01:19:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741860_1037 (size=35835) 2024-12-08T01:19:24,386 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/b/b0691074ae0140a09f74955dbafd08de 2024-12-08T01:19:24,394 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/a/24ebd2a2268f4fd2b5ed02405051594c as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/a/24ebd2a2268f4fd2b5ed02405051594c 2024-12-08T01:19:24,400 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/a/24ebd2a2268f4fd2b5ed02405051594c, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-08T01:19:24,402 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/b/b0691074ae0140a09f74955dbafd08de as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/b/b0691074ae0140a09f74955dbafd08de 2024-12-08T01:19:24,408 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/b/b0691074ae0140a09f74955dbafd08de, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-08T01:19:24,408 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 075fde603788b946bb2e6204d28ea0a9 in 76ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-08T01:19:24,427 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:19:24,428 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 075fde603788b946bb2e6204d28ea0a9 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-08T01:19:24,436 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/b/a7a8a5844f4f45b7bc39f64807fed7db is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733620759610/Put/seqid=0 2024-12-08T01:19:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741861_1038 (size=35082) 2024-12-08T01:19:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741861_1038 (size=35082) 2024-12-08T01:19:24,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741861_1038 (size=35082) 2024-12-08T01:19:24,448 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/b/a7a8a5844f4f45b7bc39f64807fed7db 2024-12-08T01:19:24,471 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/c/77f840083959467fb99eee7ff7deecb5 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733620759637/Put/seqid=0 2024-12-08T01:19:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741862_1039 (size=20825) 2024-12-08T01:19:24,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741862_1039 (size=20825) 2024-12-08T01:19:24,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741862_1039 (size=20825) 2024-12-08T01:19:24,480 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/c/77f840083959467fb99eee7ff7deecb5 2024-12-08T01:19:24,487 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/b/a7a8a5844f4f45b7bc39f64807fed7db as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/b/a7a8a5844f4f45b7bc39f64807fed7db 2024-12-08T01:19:24,495 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/b/a7a8a5844f4f45b7bc39f64807fed7db, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-08T01:19:24,496 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/c/77f840083959467fb99eee7ff7deecb5 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/c/77f840083959467fb99eee7ff7deecb5 2024-12-08T01:19:24,504 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/c/77f840083959467fb99eee7ff7deecb5, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-08T01:19:24,504 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 075fde603788b946bb2e6204d28ea0a9 in 77ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-08T01:19:24,517 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733620759680/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:24,521 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000003002 2024-12-08T01:19:24,521 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-08T01:19:24,522 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 075fde603788b946bb2e6204d28ea0a9 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-08T01:19:24,532 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/c/e376e2b23fce49588e934f69f80cef70 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733620759647/Put/seqid=0 2024-12-08T01:19:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741863_1040 (size=50301) 2024-12-08T01:19:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741863_1040 (size=50301) 2024-12-08T01:19:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741863_1040 (size=50301) 2024-12-08T01:19:24,544 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/c/e376e2b23fce49588e934f69f80cef70 2024-12-08T01:19:24,551 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e376e2b23fce49588e934f69f80cef70 2024-12-08T01:19:24,552 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/.tmp/c/e376e2b23fce49588e934f69f80cef70 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/c/e376e2b23fce49588e934f69f80cef70 2024-12-08T01:19:24,559 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e376e2b23fce49588e934f69f80cef70 2024-12-08T01:19:24,560 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/c/e376e2b23fce49588e934f69f80cef70, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-08T01:19:24,560 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 075fde603788b946bb2e6204d28ea0a9 in 39ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-08T01:19:24,561 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/0000000000000003002 2024-12-08T01:19:24,562 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:24,562 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:24,563 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T01:19:24,565 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 075fde603788b946bb2e6204d28ea0a9 2024-12-08T01:19:24,568 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenIntoWAL/075fde603788b946bb2e6204d28ea0a9/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-08T01:19:24,569 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 075fde603788b946bb2e6204d28ea0a9; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72163981, jitterRate=0.07532711327075958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T01:19:24,569 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 075fde603788b946bb2e6204d28ea0a9: Writing region info on filesystem at 1733620763883Initializing all the Stores at 1733620763884 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620763884Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620763885 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620763885Cleaning up temporary data from old regions at 1733620764562 (+677 ms)Region opened successfully at 1733620764569 (+7 ms) 2024-12-08T01:19:24,626 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 075fde603788b946bb2e6204d28ea0a9, disabling compactions & flushes 2024-12-08T01:19:24,626 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:24,627 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:24,627 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. after waiting 0 ms 2024-12-08T01:19:24,627 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:24,629 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733620759406.075fde603788b946bb2e6204d28ea0a9. 2024-12-08T01:19:24,629 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 075fde603788b946bb2e6204d28ea0a9: Waiting for close lock at 1733620764626Disabling compacts and flushes for region at 1733620764626Disabling writes for close at 1733620764627 (+1 ms)Writing region close event to WAL at 1733620764629 (+2 ms)Closed at 1733620764629 2024-12-08T01:19:24,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741857_1034 (size=175) 2024-12-08T01:19:24,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741857_1034 (size=175) 2024-12-08T01:19:24,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741857_1034 (size=175) 2024-12-08T01:19:24,636 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:24,636 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620763844) 2024-12-08T01:19:24,649 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_281409560_22 at /127.0.0.1:46228 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36935 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:36935 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34557 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_281409560_22 at /127.0.0.1:58358 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35931 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_281409560_22 at /127.0.0.1:46250 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_281409560_22 at /127.0.0.1:38948 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34557 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35931 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=10247 (was 10358) 2024-12-08T01:19:24,661 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=10245 2024-12-08T01:19:24,683 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:24,685 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:24,687 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:24,690 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79674262, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-79674262, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:24,703 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79674262/hregion-79674262.1733620764690, exclude list is [], retry=0 2024-12-08T01:19:24,706 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:24,707 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:24,707 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:24,733 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79674262/hregion-79674262.1733620764690 2024-12-08T01:19:24,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:24,733 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2dc21f25a1b06ebc52b6c7b204e2313e, NAME => 'test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:24,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741865_1042 (size=43) 2024-12-08T01:19:24,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741865_1042 (size=43) 2024-12-08T01:19:24,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741865_1042 (size=43) 2024-12-08T01:19:24,750 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:24,752 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,753 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2dc21f25a1b06ebc52b6c7b204e2313e columnFamilyName a 2024-12-08T01:19:24,754 DEBUG [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:24,754 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(327): Store=2dc21f25a1b06ebc52b6c7b204e2313e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:24,754 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,756 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2dc21f25a1b06ebc52b6c7b204e2313e columnFamilyName b 2024-12-08T01:19:24,756 DEBUG [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:24,757 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(327): Store=2dc21f25a1b06ebc52b6c7b204e2313e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:24,757 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,759 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2dc21f25a1b06ebc52b6c7b204e2313e columnFamilyName c 2024-12-08T01:19:24,759 DEBUG [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:24,759 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(327): Store=2dc21f25a1b06ebc52b6c7b204e2313e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:24,760 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,761 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,761 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,763 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,763 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,764 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:24,766 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:24,772 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:24,773 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2dc21f25a1b06ebc52b6c7b204e2313e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69902554, jitterRate=0.04162922501564026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:24,774 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2dc21f25a1b06ebc52b6c7b204e2313e: Writing region info on filesystem at 1733620764750Initializing all the Stores at 1733620764751 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620764751Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620764752 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620764752Cleaning up temporary data from old regions at 1733620764763 (+11 ms)Region opened successfully at 1733620764774 (+11 ms) 2024-12-08T01:19:24,774 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2dc21f25a1b06ebc52b6c7b204e2313e, disabling compactions & flushes 2024-12-08T01:19:24,774 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:24,774 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:24,774 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. after waiting 0 ms 2024-12-08T01:19:24,775 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:24,775 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:24,775 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2dc21f25a1b06ebc52b6c7b204e2313e: Waiting for close lock at 1733620764774Disabling compacts and flushes for region at 1733620764774Disabling writes for close at 1733620764774Writing region close event to WAL at 1733620764775 (+1 ms)Closed at 1733620764775 2024-12-08T01:19:24,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741864_1041 (size=175) 2024-12-08T01:19:24,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741864_1041 (size=175) 2024-12-08T01:19:24,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741864_1041 (size=175) 2024-12-08T01:19:24,782 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:24,782 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79674262:(num 1733620764690) 2024-12-08T01:19:24,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:24,785 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:24,798 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786, exclude list is [], retry=0 2024-12-08T01:19:24,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:24,802 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:24,803 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:24,830 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 2024-12-08T01:19:24,831 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:24,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T01:19:24,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:24,999 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 not finished, retry = 0 2024-12-08T01:19:24,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741866_1043 (size=371388) 2024-12-08T01:19:25,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741866_1043 (size=371388) 2024-12-08T01:19:25,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741866_1043 (size=371388) 2024-12-08T01:19:25,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:19:25,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T01:19:25,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T01:19:25,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:25,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-08T01:19:25,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:25,117 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786, size=362.7 K (371388bytes) 2024-12-08T01:19:25,117 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 2024-12-08T01:19:25,118 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 after 1ms 2024-12-08T01:19:25,141 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:25,143 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 took 25ms 2024-12-08T01:19:25,146 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733620764786.temp 2024-12-08T01:19:25,176 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000000001-wal.1733620764786.temp 2024-12-08T01:19:25,178 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 so closing down 2024-12-08T01:19:25,178 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:25,190 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:25,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741867_1044 (size=371388) 2024-12-08T01:19:25,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741867_1044 (size=371388) 2024-12-08T01:19:25,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741867_1044 (size=371388) 2024-12-08T01:19:25,194 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000000001-wal.1733620764786.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-12-08T01:19:25,196 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000000001-wal.1733620764786.temp to hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003000 2024-12-08T01:19:25,196 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-08T01:19:25,196 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786, journal: Splitting hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786, size=362.7 K (371388bytes) at 1733620765117Split 1024 edits, skipped 0 edits. at 1733620765156 (+39 ms)Split 2048 edits, skipped 0 edits. at 1733620765166 (+10 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000000001-wal.1733620764786.temp at 1733620765176 (+10 ms)Finishing writing output for hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 so closing down at 1733620765178 (+2 ms)3 split writer threads finished at 1733620765190 (+12 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000000001-wal.1733620764786.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1733620765194 (+4 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000000001-wal.1733620764786.temp to hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003000 at 1733620765196 (+2 ms)Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733620765196 2024-12-08T01:19:25,198 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620764786 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620764786 2024-12-08T01:19:25,199 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003000 2024-12-08T01:19:25,199 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:25,201 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:25,214 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202, exclude list is [], retry=0 2024-12-08T01:19:25,217 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:25,217 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:25,217 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:25,244 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 2024-12-08T01:19:25,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:25,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741868_1045 (size=371515) 2024-12-08T01:19:25,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741868_1045 (size=371515) 2024-12-08T01:19:25,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741868_1045 (size=371515) 2024-12-08T01:19:25,392 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202, size=362.8 K (371515bytes) 2024-12-08T01:19:25,392 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 2024-12-08T01:19:25,393 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 after 1ms 2024-12-08T01:19:25,420 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:25,422 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 took 30ms 2024-12-08T01:19:25,426 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733620765202.temp 2024-12-08T01:19:25,467 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 so closing down 2024-12-08T01:19:25,468 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:25,475 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003001-wal.1733620765202.temp 2024-12-08T01:19:25,485 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:25,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741869_1046 (size=371515) 2024-12-08T01:19:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741869_1046 (size=371515) 2024-12-08T01:19:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741869_1046 (size=371515) 2024-12-08T01:19:25,488 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003001-wal.1733620765202.temp (wrote 3000 edits, skipped 0 edits in 9 ms) 2024-12-08T01:19:25,490 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003001-wal.1733620765202.temp to hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000006000 2024-12-08T01:19:25,490 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-08T01:19:25,490 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202, journal: Splitting hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202, size=362.8 K (371515bytes) at 1733620765392Split 1024 edits, skipped 0 edits. at 1733620765435 (+43 ms)Split 2048 edits, skipped 0 edits. at 1733620765461 (+26 ms)Finishing writing output for hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 so closing down at 1733620765468 (+7 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003001-wal.1733620765202.temp at 1733620765475 (+7 ms)3 split writer threads finished at 1733620765485 (+10 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003001-wal.1733620765202.temp (wrote 3000 edits, skipped 0 edits in 9 ms) at 1733620765488 (+3 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003001-wal.1733620765202.temp to hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000006000 at 1733620765490 (+2 ms)Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733620765490 2024-12-08T01:19:25,492 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765202 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620765202 2024-12-08T01:19:25,493 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000006000 2024-12-08T01:19:25,493 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:25,495 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/test2727-manual,16010,1733620764682, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:25,506 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765496, exclude list is [], retry=0 2024-12-08T01:19:25,509 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:25,510 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:25,510 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:25,532 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733620764682/wal.1733620765496 2024-12-08T01:19:25,533 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:25,533 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2dc21f25a1b06ebc52b6c7b204e2313e, NAME => 'test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:25,533 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:25,533 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,533 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,535 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,536 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2dc21f25a1b06ebc52b6c7b204e2313e columnFamilyName a 2024-12-08T01:19:25,536 DEBUG [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:25,536 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(327): Store=2dc21f25a1b06ebc52b6c7b204e2313e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:25,536 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,538 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2dc21f25a1b06ebc52b6c7b204e2313e columnFamilyName b 2024-12-08T01:19:25,538 DEBUG [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:25,538 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(327): Store=2dc21f25a1b06ebc52b6c7b204e2313e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:25,539 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,540 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2dc21f25a1b06ebc52b6c7b204e2313e columnFamilyName c 2024-12-08T01:19:25,540 DEBUG [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:25,540 INFO [StoreOpener-2dc21f25a1b06ebc52b6c7b204e2313e-1 {}] regionserver.HStore(327): Store=2dc21f25a1b06ebc52b6c7b204e2313e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:25,541 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,541 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,544 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,545 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003000 2024-12-08T01:19:25,569 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:25,595 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003000 2024-12-08T01:19:25,596 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000006000 2024-12-08T01:19:25,619 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:25,639 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000006000 2024-12-08T01:19:25,639 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2dc21f25a1b06ebc52b6c7b204e2313e 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-08T01:19:25,657 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/a/f239eea368694c50b4cc5c931a8c581e is 41, key is test2727/a:100/1733620765248/Put/seqid=0 2024-12-08T01:19:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741871_1048 (size=84227) 2024-12-08T01:19:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741871_1048 (size=84227) 2024-12-08T01:19:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741871_1048 (size=84227) 2024-12-08T01:19:25,664 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/a/f239eea368694c50b4cc5c931a8c581e 2024-12-08T01:19:25,698 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/b/643072b8df0440e78cfdc1ab6d3518dc is 41, key is test2727/b:100/1733620765296/Put/seqid=0 2024-12-08T01:19:25,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741872_1049 (size=84609) 2024-12-08T01:19:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741872_1049 (size=84609) 2024-12-08T01:19:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741872_1049 (size=84609) 2024-12-08T01:19:25,704 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/b/643072b8df0440e78cfdc1ab6d3518dc 2024-12-08T01:19:25,736 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/c/fe63f5b54e314ae18a20c493d01c3a3e is 41, key is test2727/c:100/1733620765336/Put/seqid=0 2024-12-08T01:19:25,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741873_1050 (size=84609) 2024-12-08T01:19:25,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741873_1050 (size=84609) 2024-12-08T01:19:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741873_1050 (size=84609) 2024-12-08T01:19:25,742 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/c/fe63f5b54e314ae18a20c493d01c3a3e 2024-12-08T01:19:25,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/a/f239eea368694c50b4cc5c931a8c581e as hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/a/f239eea368694c50b4cc5c931a8c581e 2024-12-08T01:19:25,757 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/a/f239eea368694c50b4cc5c931a8c581e, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-08T01:19:25,759 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/b/643072b8df0440e78cfdc1ab6d3518dc as hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/b/643072b8df0440e78cfdc1ab6d3518dc 2024-12-08T01:19:25,765 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/b/643072b8df0440e78cfdc1ab6d3518dc, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-08T01:19:25,766 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/.tmp/c/fe63f5b54e314ae18a20c493d01c3a3e as hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/c/fe63f5b54e314ae18a20c493d01c3a3e 2024-12-08T01:19:25,773 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/c/fe63f5b54e314ae18a20c493d01c3a3e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-08T01:19:25,773 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 2dc21f25a1b06ebc52b6c7b204e2313e in 134ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-08T01:19:25,774 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000003000 2024-12-08T01:19:25,775 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/0000000000000006000 2024-12-08T01:19:25,776 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,776 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,777 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:25,778 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2dc21f25a1b06ebc52b6c7b204e2313e 2024-12-08T01:19:25,782 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/test2727/2dc21f25a1b06ebc52b6c7b204e2313e/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-08T01:19:25,784 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2dc21f25a1b06ebc52b6c7b204e2313e; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70942235, jitterRate=0.057121679186820984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:25,785 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2dc21f25a1b06ebc52b6c7b204e2313e: Writing region info on filesystem at 1733620765533Initializing all the Stores at 1733620765534 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620765534Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620765535 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620765535Obtaining lock to block concurrent updates at 1733620765639 (+104 ms)Preparing flush snapshotting stores in 2dc21f25a1b06ebc52b6c7b204e2313e at 1733620765639Finished memstore snapshotting test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733620765639Flushing stores of test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. at 1733620765639Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/a: creating writer at 1733620765639Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/a: appending metadata at 1733620765656 (+17 ms)Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/a: closing flushed file at 1733620765656Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/b: creating writer at 1733620765671 (+15 ms)Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/b: appending metadata at 1733620765696 (+25 ms)Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/b: closing flushed file at 1733620765696Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/c: creating writer at 1733620765710 (+14 ms)Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/c: appending metadata at 1733620765735 (+25 ms)Flushing 2dc21f25a1b06ebc52b6c7b204e2313e/c: closing flushed file at 1733620765735Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4151bcd2: reopening flushed file at 1733620765748 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7280d10: reopening flushed file at 1733620765757 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dccfc8f: reopening flushed file at 1733620765765 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 2dc21f25a1b06ebc52b6c7b204e2313e in 134ms, sequenceid=6000, compaction requested=false; wal=null at 1733620765773 (+8 ms)Cleaning up temporary data from old regions at 1733620765776 (+3 ms)Region opened successfully at 1733620765785 (+9 ms) 2024-12-08T01:19:25,786 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-08T01:19:25,786 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2dc21f25a1b06ebc52b6c7b204e2313e, disabling compactions & flushes 2024-12-08T01:19:25,787 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:25,787 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:25,787 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. after waiting 0 ms 2024-12-08T01:19:25,787 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:25,788 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733620764683.2dc21f25a1b06ebc52b6c7b204e2313e. 2024-12-08T01:19:25,788 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2dc21f25a1b06ebc52b6c7b204e2313e: Waiting for close lock at 1733620765786Disabling compacts and flushes for region at 1733620765786Disabling writes for close at 1733620765787 (+1 ms)Writing region close event to WAL at 1733620765788 (+1 ms)Closed at 1733620765788 2024-12-08T01:19:25,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741870_1047 (size=175) 2024-12-08T01:19:25,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741870_1047 (size=175) 2024-12-08T01:19:25,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741870_1047 (size=175) 2024-12-08T01:19:25,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:25,796 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620765496) 2024-12-08T01:19:25,811 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=406 (was 404) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:58458 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:46250 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:39076 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=10012 (was 10245) 2024-12-08T01:19:25,823 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=10010 2024-12-08T01:19:25,837 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:25,845 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:25,845 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733620765845 2024-12-08T01:19:25,876 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845 2024-12-08T01:19:25,878 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:25,879 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b5f62120bc1aeb136f0ff526d9f83662, NAME => 'testSequentialEditLogSeqNum,,1733620765838.b5f62120bc1aeb136f0ff526d9f83662.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:25,879 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733620765838.b5f62120bc1aeb136f0ff526d9f83662.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:25,879 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,879 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,880 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662 doesn't exist for region: b5f62120bc1aeb136f0ff526d9f83662 on table testSequentialEditLogSeqNum 2024-12-08T01:19:25,881 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: b5f62120bc1aeb136f0ff526d9f83662 on table testSequentialEditLogSeqNum 2024-12-08T01:19:25,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741875_1052 (size=62) 2024-12-08T01:19:25,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741875_1052 (size=62) 2024-12-08T01:19:25,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741875_1052 (size=62) 2024-12-08T01:19:25,896 INFO [StoreOpener-b5f62120bc1aeb136f0ff526d9f83662-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,898 INFO [StoreOpener-b5f62120bc1aeb136f0ff526d9f83662-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5f62120bc1aeb136f0ff526d9f83662 columnFamilyName a 2024-12-08T01:19:25,898 DEBUG [StoreOpener-b5f62120bc1aeb136f0ff526d9f83662-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:25,899 INFO [StoreOpener-b5f62120bc1aeb136f0ff526d9f83662-1 {}] regionserver.HStore(327): Store=b5f62120bc1aeb136f0ff526d9f83662/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:25,899 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,901 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,901 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,903 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b5f62120bc1aeb136f0ff526d9f83662 2024-12-08T01:19:25,906 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:25,907 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b5f62120bc1aeb136f0ff526d9f83662; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63894084, jitterRate=-0.04790395498275757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:25,908 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b5f62120bc1aeb136f0ff526d9f83662: Writing region info on filesystem at 1733620765880Initializing all the Stores at 1733620765896 (+16 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620765896Cleaning up temporary data from old regions at 1733620765901 (+5 ms)Region opened successfully at 1733620765908 (+7 ms) 2024-12-08T01:19:25,919 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b5f62120bc1aeb136f0ff526d9f83662 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-08T01:19:25,938 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/.tmp/a/854877cbbacc4b769db05839d64ba823 is 81, key is testSequentialEditLogSeqNum/a:x0/1733620765908/Put/seqid=0 2024-12-08T01:19:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741876_1053 (size=5833) 2024-12-08T01:19:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741876_1053 (size=5833) 2024-12-08T01:19:25,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741876_1053 (size=5833) 2024-12-08T01:19:25,946 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/.tmp/a/854877cbbacc4b769db05839d64ba823 2024-12-08T01:19:25,952 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/.tmp/a/854877cbbacc4b769db05839d64ba823 as hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/a/854877cbbacc4b769db05839d64ba823 2024-12-08T01:19:25,958 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/a/854877cbbacc4b769db05839d64ba823, entries=10, sequenceid=13, filesize=5.7 K 2024-12-08T01:19:25,960 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b5f62120bc1aeb136f0ff526d9f83662 in 40ms, sequenceid=13, compaction requested=false 2024-12-08T01:19:25,960 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b5f62120bc1aeb136f0ff526d9f83662: 2024-12-08T01:19:25,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:19:25,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:19:25,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:19:25,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:19:25,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-08T01:19:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741874_1051 (size=3256) 2024-12-08T01:19:25,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741874_1051 (size=3256) 2024-12-08T01:19:25,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741874_1051 (size=3256) 2024-12-08T01:19:25,981 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845, size=3.2 K (3256bytes) 2024-12-08T01:19:25,981 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845 2024-12-08T01:19:25,982 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845 after 1ms 2024-12-08T01:19:26,020 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:26,020 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845 took 39ms 2024-12-08T01:19:26,023 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845 so closing down 2024-12-08T01:19:26,023 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:26,024 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620765845.temp 2024-12-08T01:19:26,047 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000003-wal.1733620765845.temp 2024-12-08T01:19:26,047 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741877_1054 (size=2650) 2024-12-08T01:19:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741877_1054 (size=2650) 2024-12-08T01:19:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741877_1054 (size=2650) 2024-12-08T01:19:26,056 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000003-wal.1733620765845.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:26,057 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000003-wal.1733620765845.temp to hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000020 2024-12-08T01:19:26,058 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-08T01:19:26,058 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845, journal: Splitting hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845, size=3.2 K (3256bytes) at 1733620765981Finishing writing output for hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845 so closing down at 1733620766023 (+42 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000003-wal.1733620765845.temp at 1733620766047 (+24 ms)3 split writer threads finished at 1733620766047Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000003-wal.1733620765845.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733620766056 (+9 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000003-wal.1733620765845.temp to hdfs://localhost:35931/hbase/data/default/testSequentialEditLogSeqNum/b5f62120bc1aeb136f0ff526d9f83662/recovered.edits/0000000000000000020 at 1733620766057 (+1 ms)Processed 17 edits across 1 Regions in 37 ms; skipped=2; WAL=hdfs://localhost:35931/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733620765836/wal.1733620765845, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733620766058 (+1 ms) 2024-12-08T01:19:26,071 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=413 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:58458 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:46250 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=254 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=10007 (was 10010) 2024-12-08T01:19:26,081 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=254, ProcessCount=11, AvailableMemoryMB=10006 2024-12-08T01:19:26,095 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:26,098 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:26,133 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:26,136 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68048026, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-68048026, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:26,147 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68048026/hregion-68048026.1733620766136, exclude list is [], retry=0 2024-12-08T01:19:26,150 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:26,150 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:26,150 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:26,173 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68048026/hregion-68048026.1733620766136 2024-12-08T01:19:26,174 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:26,174 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2a12ee6cf5c4c5873c88d76be87aab7a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:26,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741879_1056 (size=70) 2024-12-08T01:19:26,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741879_1056 (size=70) 2024-12-08T01:19:26,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741879_1056 (size=70) 2024-12-08T01:19:26,185 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:26,186 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,188 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName a 2024-12-08T01:19:26,188 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:26,189 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:26,189 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,190 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName b 2024-12-08T01:19:26,190 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:26,191 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:26,191 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,192 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName c 2024-12-08T01:19:26,193 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:26,193 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:26,193 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,195 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,195 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,196 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:26,197 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,200 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:26,201 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a12ee6cf5c4c5873c88d76be87aab7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72590446, jitterRate=0.08168193697929382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:26,202 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a12ee6cf5c4c5873c88d76be87aab7a: Writing region info on filesystem at 1733620766185Initializing all the Stores at 1733620766186 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620766186Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620766186Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620766186Cleaning up temporary data from old regions at 1733620766195 (+9 ms)Region opened successfully at 1733620766202 (+7 ms) 2024-12-08T01:19:26,202 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2a12ee6cf5c4c5873c88d76be87aab7a, disabling compactions & flushes 2024-12-08T01:19:26,202 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:26,202 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:26,202 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. after waiting 0 ms 2024-12-08T01:19:26,203 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:26,203 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:26,203 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2a12ee6cf5c4c5873c88d76be87aab7a: Waiting for close lock at 1733620766202Disabling compacts and flushes for region at 1733620766202Disabling writes for close at 1733620766202Writing region close event to WAL at 1733620766203 (+1 ms)Closed at 1733620766203 2024-12-08T01:19:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741878_1055 (size=175) 2024-12-08T01:19:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741878_1055 (size=175) 2024-12-08T01:19:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741878_1055 (size=175) 2024-12-08T01:19:26,210 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:26,210 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68048026:(num 1733620766136) 2024-12-08T01:19:26,210 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:26,212 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:26,227 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212, exclude list is [], retry=0 2024-12-08T01:19:26,229 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:26,229 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:26,230 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:26,254 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 2024-12-08T01:19:26,254 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:26,254 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a12ee6cf5c4c5873c88d76be87aab7a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:26,255 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:26,255 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,255 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,256 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,257 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName a 2024-12-08T01:19:26,257 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:26,258 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:26,258 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,259 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName b 2024-12-08T01:19:26,259 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:26,259 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:26,259 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,260 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName c 2024-12-08T01:19:26,260 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:26,261 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:26,261 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,261 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,263 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,263 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,264 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:26,265 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:26,266 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a12ee6cf5c4c5873c88d76be87aab7a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60044427, jitterRate=-0.10526831448078156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:26,266 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a12ee6cf5c4c5873c88d76be87aab7a: Writing region info on filesystem at 1733620766255Initializing all the Stores at 1733620766256 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620766256Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620766256Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620766256Cleaning up temporary data from old regions at 1733620766263 (+7 ms)Region opened successfully at 1733620766266 (+3 ms) 2024-12-08T01:19:26,270 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733620766269/Put/seqid=0 2024-12-08T01:19:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741881_1058 (size=4826) 2024-12-08T01:19:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741881_1058 (size=4826) 2024-12-08T01:19:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741881_1058 (size=4826) 2024-12-08T01:19:26,278 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35931/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 2a12ee6cf5c4c5873c88d76be87aab7a/a 2024-12-08T01:19:26,286 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-08T01:19:26,286 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:19:26,287 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2a12ee6cf5c4c5873c88d76be87aab7a: 2024-12-08T01:19:26,288 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_ 2024-12-08T01:19:26,289 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35931/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2a12ee6cf5c4c5873c88d76be87aab7a/a as hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_ - updating store file list. 2024-12-08T01:19:26,294 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:26,294 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_ into 2a12ee6cf5c4c5873c88d76be87aab7a/a 2024-12-08T01:19:26,294 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35931/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2a12ee6cf5c4c5873c88d76be87aab7a/a (new location: hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_) 2024-12-08T01:19:26,330 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212, size=0 (0bytes) 2024-12-08T01:19:26,330 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 might be still open, length is 0 2024-12-08T01:19:26,330 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 2024-12-08T01:19:26,331 WARN [IPC Server handler 2 on default port 35931 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-08T01:19:26,331 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 after 1ms 2024-12-08T01:19:28,822 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:46460 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39611:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46460 dst: /127.0.0.1:39611 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39611 remote=/127.0.0.1:46460]. Total timeout mills is 60000, 57479 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:28,822 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:58568 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58568 dst: /127.0.0.1:41533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:28,822 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:39138 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39138 dst: /127.0.0.1:39867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:28,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741880_1059 (size=606) 2024-12-08T01:19:28,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741880_1059 (size=606) 2024-12-08T01:19:30,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 after 4002ms 2024-12-08T01:19:30,364 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:30,364 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 took 4034ms 2024-12-08T01:19:30,366 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212; continuing. 2024-12-08T01:19:30,367 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 so closing down 2024-12-08T01:19:30,367 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:30,368 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733620766212.temp 2024-12-08T01:19:30,392 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005-wal.1733620766212.temp 2024-12-08T01:19:30,392 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741882_1060 (size=329) 2024-12-08T01:19:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741882_1060 (size=329) 2024-12-08T01:19:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741882_1060 (size=329) 2024-12-08T01:19:30,399 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005-wal.1733620766212.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:30,400 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005-wal.1733620766212.temp to hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005 2024-12-08T01:19:30,400 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:19:30,400 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212, journal: Splitting hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212, size=0 (0bytes) at 1733620766330Finishing writing output for hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 so closing down at 1733620770367 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005-wal.1733620766212.temp at 1733620770392 (+25 ms)3 split writer threads finished at 1733620770392Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005-wal.1733620766212.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733620770399 (+7 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005-wal.1733620766212.temp to hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005 at 1733620770400 (+1 ms)Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212, size=0, length=0, corrupted=false, cancelled=false at 1733620770400 2024-12-08T01:19:30,402 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620766212 2024-12-08T01:19:30,403 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005 2024-12-08T01:19:30,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:30,405 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:30,417 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620770405, exclude list is [], retry=0 2024-12-08T01:19:30,419 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:30,419 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:30,420 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:30,442 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620770405 2024-12-08T01:19:30,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:30,443 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a12ee6cf5c4c5873c88d76be87aab7a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:30,443 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:30,443 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,443 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,445 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,445 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName a 2024-12-08T01:19:30,446 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:30,452 DEBUG [StoreFileOpener-2a12ee6cf5c4c5873c88d76be87aab7a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:30,452 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/dd70dd0518354d2b979104bec0cdb0bf_SeqId_3_ 2024-12-08T01:19:30,452 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:30,452 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,453 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName b 2024-12-08T01:19:30,453 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:30,454 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:30,454 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,455 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a12ee6cf5c4c5873c88d76be87aab7a columnFamilyName c 2024-12-08T01:19:30,455 DEBUG [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:30,456 INFO [StoreOpener-2a12ee6cf5c4c5873c88d76be87aab7a-1 {}] regionserver.HStore(327): Store=2a12ee6cf5c4c5873c88d76be87aab7a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:30,456 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,457 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,459 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,459 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005 2024-12-08T01:19:30,483 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:30,484 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005 2024-12-08T01:19:30,485 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2a12ee6cf5c4c5873c88d76be87aab7a 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-08T01:19:30,497 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/.tmp/a/d449840697b94f1097d13db24b16486f is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733620766300/Put/seqid=0 2024-12-08T01:19:30,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741884_1062 (size=5149) 2024-12-08T01:19:30,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741884_1062 (size=5149) 2024-12-08T01:19:30,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741884_1062 (size=5149) 2024-12-08T01:19:30,503 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/.tmp/a/d449840697b94f1097d13db24b16486f 2024-12-08T01:19:30,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/.tmp/a/d449840697b94f1097d13db24b16486f as hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/d449840697b94f1097d13db24b16486f 2024-12-08T01:19:30,515 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/a/d449840697b94f1097d13db24b16486f, entries=1, sequenceid=5, filesize=5.0 K 2024-12-08T01:19:30,515 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2a12ee6cf5c4c5873c88d76be87aab7a in 31ms, sequenceid=5, compaction requested=false; wal=null 2024-12-08T01:19:30,516 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/0000000000000000005 2024-12-08T01:19:30,517 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,517 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,518 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:30,519 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a12ee6cf5c4c5873c88d76be87aab7a 2024-12-08T01:19:30,521 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2a12ee6cf5c4c5873c88d76be87aab7a/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-08T01:19:30,522 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a12ee6cf5c4c5873c88d76be87aab7a; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74398009, jitterRate=0.10861672461032867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:30,522 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a12ee6cf5c4c5873c88d76be87aab7a: Writing region info on filesystem at 1733620770443Initializing all the Stores at 1733620770444 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620770444Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620770444Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620770444Obtaining lock to block concurrent updates at 1733620770485 (+41 ms)Preparing flush snapshotting stores in 2a12ee6cf5c4c5873c88d76be87aab7a at 1733620770485Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733620770485Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. at 1733620770485Flushing 2a12ee6cf5c4c5873c88d76be87aab7a/a: creating writer at 1733620770485Flushing 2a12ee6cf5c4c5873c88d76be87aab7a/a: appending metadata at 1733620770496 (+11 ms)Flushing 2a12ee6cf5c4c5873c88d76be87aab7a/a: closing flushed file at 1733620770496Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5163485a: reopening flushed file at 1733620770508 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2a12ee6cf5c4c5873c88d76be87aab7a in 31ms, sequenceid=5, compaction requested=false; wal=null at 1733620770515 (+7 ms)Cleaning up temporary data from old regions at 1733620770517 (+2 ms)Region opened successfully at 1733620770522 (+5 ms) 2024-12-08T01:19:30,526 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2a12ee6cf5c4c5873c88d76be87aab7a, disabling compactions & flushes 2024-12-08T01:19:30,526 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:30,526 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:30,526 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. after waiting 0 ms 2024-12-08T01:19:30,526 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:30,527 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733620766096.2a12ee6cf5c4c5873c88d76be87aab7a. 2024-12-08T01:19:30,527 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2a12ee6cf5c4c5873c88d76be87aab7a: Waiting for close lock at 1733620770526Disabling compacts and flushes for region at 1733620770526Disabling writes for close at 1733620770526Writing region close event to WAL at 1733620770527 (+1 ms)Closed at 1733620770527 2024-12-08T01:19:30,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741883_1061 (size=175) 2024-12-08T01:19:30,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741883_1061 (size=175) 2024-12-08T01:19:30,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741883_1061 (size=175) 2024-12-08T01:19:30,532 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:30,532 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620770405) 2024-12-08T01:19:30,543 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=417 (was 413) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35931 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35931 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_94121424_22 at /127.0.0.1:50874 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_94121424_22 at /127.0.0.1:50956 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43593 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_94121424_22 at /127.0.0.1:45800 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43593 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 254), ProcessCount=11 (was 11), AvailableMemoryMB=9994 (was 10006) 2024-12-08T01:19:30,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=417, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=9993 2024-12-08T01:19:30,573 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:30,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T01:19:30,580 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 96982154c400,39725,1733620754685 2024-12-08T01:19:30,581 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f0be686 2024-12-08T01:19:30,582 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T01:19:30,584 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T01:19:30,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T01:19:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-08T01:19:30,595 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T01:19:30,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-08T01:19:30,597 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:30,599 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T01:19:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:19:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741885_1063 (size=694) 2024-12-08T01:19:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741885_1063 (size=694) 2024-12-08T01:19:30,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741885_1063 (size=694) 2024-12-08T01:19:30,611 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae 2024-12-08T01:19:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741886_1064 (size=77) 2024-12-08T01:19:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741886_1064 (size=77) 2024-12-08T01:19:30,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741886_1064 (size=77) 2024-12-08T01:19:30,621 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:30,621 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:30,621 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,621 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,621 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:30,621 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,621 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,621 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620770621Disabling compacts and flushes for region at 1733620770621Disabling writes for close at 1733620770621Writing region close event to WAL at 1733620770621Closed at 1733620770621 2024-12-08T01:19:30,623 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T01:19:30,627 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733620770623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733620770623"}]},"ts":"1733620770623"} 2024-12-08T01:19:30,630 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T01:19:30,632 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T01:19:30,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620770632"}]},"ts":"1733620770632"} 2024-12-08T01:19:30,637 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-08T01:19:30,638 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-08T01:19:30,639 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:19:30,639 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:19:30,639 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-08T01:19:30,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:19:30,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN}] 2024-12-08T01:19:30,643 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN 2024-12-08T01:19:30,644 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN; state=OFFLINE, location=96982154c400,35801,1733620755391; forceNewPlan=false, retain=false 2024-12-08T01:19:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:19:30,797 INFO [96982154c400:39725 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T01:19:30,798 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPENING, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:30,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN because future has completed 2024-12-08T01:19:30,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391}] 2024-12-08T01:19:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:19:30,962 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,963 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:30,963 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,963 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:30,963 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,964 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,965 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,968 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf1 2024-12-08T01:19:30,968 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:30,969 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:30,969 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,971 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf2 2024-12-08T01:19:30,971 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:30,972 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:30,972 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,973 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,974 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,975 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,975 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,977 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:19:30,979 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,982 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:30,983 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7b9fc8dd8c927db706d5818cc32f0abc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73833346, jitterRate=0.10020259022712708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:19:30,983 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:30,984 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7b9fc8dd8c927db706d5818cc32f0abc: Running coprocessor pre-open hook at 1733620770964Writing region info on filesystem at 1733620770964Initializing all the Stores at 1733620770965 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620770965Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620770965Cleaning up temporary data from old regions at 1733620770975 (+10 ms)Running coprocessor post-open hooks at 1733620770983 (+8 ms)Region opened successfully at 1733620770984 (+1 ms) 2024-12-08T01:19:30,987 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., pid=6, masterSystemTime=1733620770956 2024-12-08T01:19:30,990 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,990 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:30,991 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPEN, openSeqNum=2, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:30,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391 because future has completed 2024-12-08T01:19:30,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T01:19:30,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391 in 194 msec 2024-12-08T01:19:31,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T01:19:31,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN in 358 msec 2024-12-08T01:19:31,004 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T01:19:31,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733620771004"}]},"ts":"1733620771004"} 2024-12-08T01:19:31,006 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-08T01:19:31,008 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T01:19:31,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 419 msec 2024-12-08T01:19:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T01:19:31,235 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-08T01:19:31,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-08T01:19:31,238 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T01:19:31,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-08T01:19:31,246 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-08T01:19:31,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-08T01:19:31,258 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=2] 2024-12-08T01:19:31,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7b9fc8dd8c927db706d5818cc32f0abc, source=96982154c400,35801,1733620755391, destination=96982154c400,37871,1733620755662, warming up region on 96982154c400,37871,1733620755662 2024-12-08T01:19:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:19:31,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7b9fc8dd8c927db706d5818cc32f0abc, source=96982154c400,35801,1733620755391, destination=96982154c400,37871,1733620755662, running balancer 2024-12-08T01:19:31,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55117, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:19:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE 2024-12-08T01:19:31,279 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE 2024-12-08T01:19:31,281 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=CLOSING, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:31,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:31,284 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE because future has completed 2024-12-08T01:19:31,284 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:19:31,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391}] 2024-12-08T01:19:31,285 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf1 2024-12-08T01:19:31,285 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:31,286 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:31,286 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,287 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf2 2024-12-08T01:19:31,287 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:31,288 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:31,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620771288Disabling compacts and flushes for region at 1733620771288Disabling writes for close at 1733620771288Writing region close event to WAL at 1733620771288Closed at 1733620771288 2024-12-08T01:19:31,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-08T01:19:31,442 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,442 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:19:31,443 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:31,443 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,443 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,443 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:31,443 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,443 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 7b9fc8dd8c927db706d5818cc32f0abc 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-08T01:19:31,458 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/98589aaf86b44709a20b2c924fbfebfe is 35, key is r1/cf1:q/1733620771260/Put/seqid=0 2024-12-08T01:19:31,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741887_1065 (size=4783) 2024-12-08T01:19:31,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741887_1065 (size=4783) 2024-12-08T01:19:31,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741887_1065 (size=4783) 2024-12-08T01:19:31,466 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/98589aaf86b44709a20b2c924fbfebfe 2024-12-08T01:19:31,473 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/98589aaf86b44709a20b2c924fbfebfe as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe 2024-12-08T01:19:31,480 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe, entries=1, sequenceid=5, filesize=4.7 K 2024-12-08T01:19:31,482 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 7b9fc8dd8c927db706d5818cc32f0abc in 38ms, sequenceid=5, compaction requested=false 2024-12-08T01:19:31,482 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-08T01:19:31,487 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-08T01:19:31,490 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,490 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620771443Running coprocessor pre-close hooks at 1733620771443Disabling compacts and flushes for region at 1733620771443Disabling writes for close at 1733620771443Obtaining lock to block concurrent updates at 1733620771443Preparing flush snapshotting stores in 7b9fc8dd8c927db706d5818cc32f0abc at 1733620771443Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733620771443Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. at 1733620771444 (+1 ms)Flushing 7b9fc8dd8c927db706d5818cc32f0abc/cf1: creating writer at 1733620771444Flushing 7b9fc8dd8c927db706d5818cc32f0abc/cf1: appending metadata at 1733620771458 (+14 ms)Flushing 7b9fc8dd8c927db706d5818cc32f0abc/cf1: closing flushed file at 1733620771458Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1820c7f7: reopening flushed file at 1733620771472 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 7b9fc8dd8c927db706d5818cc32f0abc in 38ms, sequenceid=5, compaction requested=false at 1733620771482 (+10 ms)Writing region close event to WAL at 1733620771483 (+1 ms)Running coprocessor post-close hooks at 1733620771488 (+5 ms)Closed at 1733620771490 (+2 ms) 2024-12-08T01:19:31,490 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7b9fc8dd8c927db706d5818cc32f0abc move to 96982154c400,37871,1733620755662 record at close sequenceid=5 2024-12-08T01:19:31,494 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,494 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=CLOSED 2024-12-08T01:19:31,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391 because future has completed 2024-12-08T01:19:31,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T01:19:31,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391 in 214 msec 2024-12-08T01:19:31,504 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE; state=CLOSED, location=96982154c400,37871,1733620755662; forceNewPlan=false, retain=false 2024-12-08T01:19:31,654 INFO [96982154c400:39725 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T01:19:31,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPENING, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:31,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE because future has completed 2024-12-08T01:19:31,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662}] 2024-12-08T01:19:31,820 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,820 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:31,820 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,821 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:31,821 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,821 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,824 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,826 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf1 2024-12-08T01:19:31,826 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:31,834 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe 2024-12-08T01:19:31,834 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:31,834 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,835 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf2 2024-12-08T01:19:31,835 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:31,836 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:31,836 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,837 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,839 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,840 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,840 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,841 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:19:31,842 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,843 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7b9fc8dd8c927db706d5818cc32f0abc; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60490441, jitterRate=-0.09862218797206879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:19:31,844 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:31,845 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7b9fc8dd8c927db706d5818cc32f0abc: Running coprocessor pre-open hook at 1733620771821Writing region info on filesystem at 1733620771821Initializing all the Stores at 1733620771824 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620771824Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620771824Cleaning up temporary data from old regions at 1733620771840 (+16 ms)Running coprocessor post-open hooks at 1733620771844 (+4 ms)Region opened successfully at 1733620771844 2024-12-08T01:19:31,846 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., pid=9, masterSystemTime=1733620771815 2024-12-08T01:19:31,849 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,849 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:31,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPEN, openSeqNum=9, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:31,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662 because future has completed 2024-12-08T01:19:31,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-08T01:19:31,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662 in 192 msec 2024-12-08T01:19:31,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE in 580 msec 2024-12-08T01:19:31,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:19:31,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:19:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:47268 deadline: 1733620831890, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=37871 startCode=1733620755662. As of locationSeqNum=5. 2024-12-08T01:19:31,911 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=37871 startCode=1733620755662. As of locationSeqNum=5. 2024-12-08T01:19:31,912 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=37871 startCode=1733620755662. As of locationSeqNum=5. 2024-12-08T01:19:31,912 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,37871,1733620755662, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=37871 startCode=1733620755662. As of locationSeqNum=5. 2024-12-08T01:19:32,022 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:19:32,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:19:32,032 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b9fc8dd8c927db706d5818cc32f0abc 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-08T01:19:32,049 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/dc910b17bd564afe87fcd29ab788d197 is 29, key is r1/cf1:/1733620772024/DeleteFamily/seqid=0 2024-12-08T01:19:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741888_1066 (size=4906) 2024-12-08T01:19:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741888_1066 (size=4906) 2024-12-08T01:19:32,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741888_1066 (size=4906) 2024-12-08T01:19:32,057 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,063 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf2/286e2fbd446641859b396362282cf7c5 is 29, key is r1/cf2:/1733620772024/DeleteFamily/seqid=0 2024-12-08T01:19:32,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741889_1067 (size=4906) 2024-12-08T01:19:32,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741889_1067 (size=4906) 2024-12-08T01:19:32,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741889_1067 (size=4906) 2024-12-08T01:19:32,083 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf2/286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,089 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,090 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/dc910b17bd564afe87fcd29ab788d197 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,096 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,096 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197, entries=1, sequenceid=12, filesize=4.8 K 2024-12-08T01:19:32,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf2/286e2fbd446641859b396362282cf7c5 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,103 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,103 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5, entries=1, sequenceid=12, filesize=4.8 K 2024-12-08T01:19:32,104 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 7b9fc8dd8c927db706d5818cc32f0abc in 72ms, sequenceid=12, compaction requested=false 2024-12-08T01:19:32,104 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7b9fc8dd8c927db706d5818cc32f0abc: 2024-12-08T01:19:32,106 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T01:19:32,108 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7b9fc8dd8c927db706d5818cc32f0abc/cf1 is initiating major compaction (all files) 2024-12-08T01:19:32,108 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:19:32,108 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:32,109 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7b9fc8dd8c927db706d5818cc32f0abc/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,109 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe, hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197] into tmpdir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp, totalSize=9.5 K 2024-12-08T01:19:32,110 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 98589aaf86b44709a20b2c924fbfebfe, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733620771260 2024-12-08T01:19:32,111 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dc910b17bd564afe87fcd29ab788d197, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-08T01:19:32,123 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7b9fc8dd8c927db706d5818cc32f0abc#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:19:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741890_1068 (size=4626) 2024-12-08T01:19:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741890_1068 (size=4626) 2024-12-08T01:19:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741890_1068 (size=4626) 2024-12-08T01:19:32,136 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf1/20020cb61c474860b3cd47882235a65e as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/20020cb61c474860b3cd47882235a65e 2024-12-08T01:19:32,149 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 7b9fc8dd8c927db706d5818cc32f0abc/cf1 of 7b9fc8dd8c927db706d5818cc32f0abc into 20020cb61c474860b3cd47882235a65e(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:19:32,149 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7b9fc8dd8c927db706d5818cc32f0abc: 2024-12-08T01:19:32,149 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-08T01:19:32,149 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7b9fc8dd8c927db706d5818cc32f0abc/cf2 is initiating major compaction (all files) 2024-12-08T01:19:32,149 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T01:19:32,149 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T01:19:32,149 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7b9fc8dd8c927db706d5818cc32f0abc/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,150 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5] into tmpdir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp, totalSize=4.8 K 2024-12-08T01:19:32,150 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 286e2fbd446641859b396362282cf7c5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-08T01:19:32,156 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7b9fc8dd8c927db706d5818cc32f0abc#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T01:19:32,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741891_1069 (size=4592) 2024-12-08T01:19:32,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741891_1069 (size=4592) 2024-12-08T01:19:32,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741891_1069 (size=4592) 2024-12-08T01:19:32,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/.tmp/cf2/bb70b41d5698418bab58658e035b2b04 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/bb70b41d5698418bab58658e035b2b04 2024-12-08T01:19:32,176 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 7b9fc8dd8c927db706d5818cc32f0abc/cf2 of 7b9fc8dd8c927db706d5818cc32f0abc into bb70b41d5698418bab58658e035b2b04(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:19:32,176 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7b9fc8dd8c927db706d5818cc32f0abc: 2024-12-08T01:19:32,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7b9fc8dd8c927db706d5818cc32f0abc, source=96982154c400,37871,1733620755662, destination=96982154c400,35801,1733620755391, warming up region on 96982154c400,35801,1733620755391 2024-12-08T01:19:32,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7b9fc8dd8c927db706d5818cc32f0abc, source=96982154c400,37871,1733620755662, destination=96982154c400,35801,1733620755391, running balancer 2024-12-08T01:19:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE 2024-12-08T01:19:32,182 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE 2024-12-08T01:19:32,183 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=CLOSING, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:32,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:32,185 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,186 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf1 2024-12-08T01:19:32,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE because future has completed 2024-12-08T01:19:32,186 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:32,186 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T01:19:32,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662}] 2024-12-08T01:19:32,194 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/20020cb61c474860b3cd47882235a65e 2024-12-08T01:19:32,199 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe 2024-12-08T01:19:32,204 INFO [StoreFileOpener-7b9fc8dd8c927db706d5818cc32f0abc-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,204 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,204 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:32,205 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,206 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf2 2024-12-08T01:19:32,206 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:32,212 INFO [StoreFileOpener-7b9fc8dd8c927db706d5818cc32f0abc-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,212 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,217 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/bb70b41d5698418bab58658e035b2b04 2024-12-08T01:19:32,218 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:32,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35801 {}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620772218Disabling compacts and flushes for region at 1733620772218Disabling writes for close at 1733620772218Writing region close event to WAL at 1733620772219 (+1 ms)Closed at 1733620772219 2024-12-08T01:19:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-08T01:19:32,340 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,340 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T01:19:32,341 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:32,341 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,341 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,341 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:32,341 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,341 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe, hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197] to archive 2024-12-08T01:19:32,344 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:19:32,348 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/98589aaf86b44709a20b2c924fbfebfe 2024-12-08T01:19:32,349 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197 to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/dc910b17bd564afe87fcd29ab788d197 2024-12-08T01:19:32,360 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5] to archive 2024-12-08T01:19:32,361 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:19:32,363 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5 to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/286e2fbd446641859b396362282cf7c5 2024-12-08T01:19:32,368 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-08T01:19:32,369 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,369 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620772341Running coprocessor pre-close hooks at 1733620772341Disabling compacts and flushes for region at 1733620772341Disabling writes for close at 1733620772341Writing region close event to WAL at 1733620772364 (+23 ms)Running coprocessor post-close hooks at 1733620772369 (+5 ms)Closed at 1733620772369 2024-12-08T01:19:32,369 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 7b9fc8dd8c927db706d5818cc32f0abc move to 96982154c400,35801,1733620755391 record at close sequenceid=12 2024-12-08T01:19:32,371 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,372 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=CLOSED 2024-12-08T01:19:32,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662 because future has completed 2024-12-08T01:19:32,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-08T01:19:32,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662 in 190 msec 2024-12-08T01:19:32,380 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE; state=CLOSED, location=96982154c400,35801,1733620755391; forceNewPlan=false, retain=false 2024-12-08T01:19:32,530 INFO [96982154c400:39725 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-08T01:19:32,531 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPENING, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:32,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE because future has completed 2024-12-08T01:19:32,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391}] 2024-12-08T01:19:32,695 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,695 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:32,696 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,696 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:32,696 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,696 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,698 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,700 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf1 2024-12-08T01:19:32,700 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:32,708 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/20020cb61c474860b3cd47882235a65e 2024-12-08T01:19:32,708 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:32,708 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,709 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf2 2024-12-08T01:19:32,709 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:32,715 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/bb70b41d5698418bab58658e035b2b04 2024-12-08T01:19:32,716 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:32,716 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,717 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,718 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,718 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,719 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,719 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:19:32,721 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,721 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 7b9fc8dd8c927db706d5818cc32f0abc; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62193750, jitterRate=-0.07324090600013733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:19:32,721 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,722 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 7b9fc8dd8c927db706d5818cc32f0abc: Running coprocessor pre-open hook at 1733620772697Writing region info on filesystem at 1733620772697Initializing all the Stores at 1733620772698 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620772698Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620772698Cleaning up temporary data from old regions at 1733620772719 (+21 ms)Running coprocessor post-open hooks at 1733620772721 (+2 ms)Region opened successfully at 1733620772722 (+1 ms) 2024-12-08T01:19:32,723 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., pid=12, masterSystemTime=1733620772688 2024-12-08T01:19:32,725 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,726 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,726 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPEN, openSeqNum=18, regionLocation=96982154c400,35801,1733620755391 2024-12-08T01:19:32,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391 because future has completed 2024-12-08T01:19:32,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-08T01:19:32,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,35801,1733620755391 in 196 msec 2024-12-08T01:19:32,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, REOPEN/MOVE in 553 msec 2024-12-08T01:19:32,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:19:32,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:19:32,787 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 96982154c400,35801,1733620755391: testing ***** 2024-12-08T01:19:32,787 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-08T01:19:32,789 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-08T01:19:32,790 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-08T01:19:32,792 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-08T01:19:32,792 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-08T01:19:32,801 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 470144536 }, "NonHeapMemoryUsage": { "committed": 172228608, "init": 7667712, "max": -1, "used": 169519280 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "96982154c400", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "96982154c400", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54213", "tag.serverName": "96982154c400,37871,1733620755662", "tag.clusterId": "48c17904-c0cd-482c-afe5-f922c94d85d2", "tag.Context": "regionserver", "tag.Hostname": "96982154c400", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733620755662, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-08T01:19:32,804 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39725 {}] master.MasterRpcServices(700): 96982154c400,35801,1733620755391 reported a fatal error: ***** ABORTING region server 96982154c400,35801,1733620755391: testing ***** 2024-12-08T01:19:32,806 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,35801,1733620755391' ***** 2024-12-08T01:19:32,806 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-08T01:19:32,807 INFO [RS:0;96982154c400:35801 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:19:32,807 INFO [RS:0;96982154c400:35801 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-08T01:19:32,807 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:19:32,807 INFO [RS:0;96982154c400:35801 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-08T01:19:32,807 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:32,807 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(956): aborting server 96982154c400,35801,1733620755391 2024-12-08T01:19:32,808 INFO [RS:0;96982154c400:35801 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:32,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37871 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:34378 deadline: 1733620832808, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35801 startCode=1733620755391. As of locationSeqNum=12. 2024-12-08T01:19:32,808 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:32,808 INFO [RS:0;96982154c400:35801 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;96982154c400:35801. 2024-12-08T01:19:32,808 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,808 DEBUG [RS:0;96982154c400:35801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:32,808 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,808 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:32,808 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,808 DEBUG [RS:0;96982154c400:35801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:32,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,37871,1733620755662, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,37871,1733620755662, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35801 startCode=1733620755391. As of locationSeqNum=12. 2024-12-08T01:19:32,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,37871,1733620755662, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35801 startCode=1733620755391. As of locationSeqNum=12. 2024-12-08T01:19:32,809 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,37871,1733620755662, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=96982154c400 port=35801 startCode=1733620755391. As of locationSeqNum=12. 2024-12-08T01:19:32,809 INFO [RS:0;96982154c400:35801 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:19:32,809 INFO [RS:0;96982154c400:35801 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:19:32,809 INFO [RS:0;96982154c400:35801 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:19:32,809 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T01:19:32,810 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,810 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620772808Running coprocessor pre-close hooks at 1733620772808Disabling compacts and flushes for region at 1733620772808Disabling writes for close at 1733620772808Writing region close event to WAL at 1733620772809 (+1 ms)Running coprocessor post-close hooks at 1733620772809Closed at 1733620772810 (+1 ms) 2024-12-08T01:19:32,810 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:32,810 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T01:19:32,810 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:19:32,810 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T01:19:32,810 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:19:32,810 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:19:32,811 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:19:32,811 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:19:32,811 DEBUG [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T01:19:32,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35801,1733620755391 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:32,811 ERROR [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-08T01:19:32,812 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:19:32,812 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:32,812 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620772810Running coprocessor pre-close hooks at 1733620772810Disabling compacts and flushes for region at 1733620772810Disabling writes for close at 1733620772811 (+1 ms)Writing region close event to WAL at 1733620772811Running coprocessor post-close hooks at 1733620772812 (+1 ms)Closed at 1733620772812 2024-12-08T01:19:32,812 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:32,812 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35801,1733620755391 aborting 2024-12-08T01:19:32,812 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35801,1733620755391 aborting 2024-12-08T01:19:32,812 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 from cache 2024-12-08T01:19:32,818 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:32,922 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:32,923 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1] 2024-12-08T01:19:32,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35801,1733620755391 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:32,924 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35801,1733620755391 aborting 2024-12-08T01:19:32,924 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 96982154c400,35801,1733620755391 aborting 2024-12-08T01:19:32,924 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 from cache 2024-12-08T01:19:33,011 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(976): stopping server 96982154c400,35801,1733620755391; all regions closed. 2024-12-08T01:19:33,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741836_1012 (size=7323) 2024-12-08T01:19:33,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741836_1012 (size=7323) 2024-12-08T01:19:33,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741836_1012 (size=7323) 2024-12-08T01:19:33,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741834_1010 (size=2288) 2024-12-08T01:19:33,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741834_1010 (size=2288) 2024-12-08T01:19:33,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741834_1010 (size=2288) 2024-12-08T01:19:33,018 DEBUG [RS:0;96982154c400:35801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:33,018 INFO [RS:0;96982154c400:35801 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:33,018 INFO [RS:0;96982154c400:35801 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:33,018 INFO [RS:0;96982154c400:35801 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:33,019 INFO [RS:0;96982154c400:35801 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:33,019 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:33,019 INFO [RS:0;96982154c400:35801 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35801 2024-12-08T01:19:33,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,35801,1733620755391 2024-12-08T01:19:33,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:19:33,051 INFO [RS:0;96982154c400:35801 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:33,052 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,35801,1733620755391] 2024-12-08T01:19:33,072 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,35801,1733620755391 already deleted, retry=false 2024-12-08T01:19:33,072 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 96982154c400,35801,1733620755391 on 96982154c400,39725,1733620754685 2024-12-08T01:19:33,077 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 96982154c400,35801,1733620755391, splitWal=true, meta=true 2024-12-08T01:19:33,079 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 96982154c400,35801,1733620755391 (carryingMeta=true) 96982154c400,35801,1733620755391/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3dc62667[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-08T01:19:33,080 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 96982154c400,35801,1733620755391, splitWal=true, meta=true 2024-12-08T01:19:33,081 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 96982154c400,35801,1733620755391, splitWal=true, meta=true, isMeta: true 2024-12-08T01:19:33,083 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting 2024-12-08T01:19:33,084 INFO [PEWorker-1 {}] master.SplitWALManager(105): 96982154c400,35801,1733620755391 WAL count=1, meta=true 2024-12-08T01:19:33,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta}] 2024-12-08T01:19:33,091 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=96982154c400,43235,1733620755742 2024-12-08T01:19:33,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, worker=96982154c400,43235,1733620755742}] 2024-12-08T01:19:33,132 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:33,133 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1] 2024-12-08T01:19:33,134 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 96982154c400:35801 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 96982154c400/172.17.0.2:35801 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:33,135 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1, error=java.net.ConnectException: Call to address=96982154c400:35801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 96982154c400/172.17.0.2:35801 2024-12-08T01:19:33,135 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-08T01:19:33,135 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 from cache 2024-12-08T01:19:33,136 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 96982154c400:35801 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 96982154c400/172.17.0.2:35801 2024-12-08T01:19:33,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:33,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35801-0x10003016cce0001, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:33,163 INFO [RS:0;96982154c400:35801 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:33,163 INFO [RS:0;96982154c400:35801 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,35801,1733620755391; zookeeper connection closed. 2024-12-08T01:19:33,163 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75a798c7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75a798c7 2024-12-08T01:19:33,255 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T01:19:33,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52245, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T01:19:33,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43235 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-08T01:19:33,277 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, size=7.2 K (7323bytes) 2024-12-08T01:19:33,277 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta 2024-12-08T01:19:33,277 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta after 0ms 2024-12-08T01:19:33,299 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:33,299 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta took 22ms 2024-12-08T01:19:33,305 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-08T01:19:33,305 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta so closing down 2024-12-08T01:19:33,306 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:33,307 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp 2024-12-08T01:19:33,329 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp 2024-12-08T01:19:33,330 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741892_1070 (size=7071) 2024-12-08T01:19:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741892_1070 (size=7071) 2024-12-08T01:19:33,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741892_1070 (size=7071) 2024-12-08T01:19:33,337 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:33,339 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:19:33,339 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-08T01:19:33,339 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, journal: Splitting hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, size=7.2 K (7323bytes) at 1733620773277Finishing writing output for hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta so closing down at 1733620773305 (+28 ms)Creating recovered edits writer path=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp at 1733620773330 (+25 ms)3 split writer threads finished at 1733620773330Closed recovered edits writer path=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733620773337 (+7 ms)Rename recovered edits hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta.temp to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733620773339 (+2 ms)Processed 16 edits across 1 Regions in 40 ms; skipped=1; WAL=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733620773339 2024-12-08T01:19:33,339 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta 2024-12-08T01:19:33,340 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-08T01:19:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-08T01:19:33,346 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs 2024-12-08T01:19:33,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-08T01:19:33,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, worker=96982154c400,43235,1733620755742 in 254 msec 2024-12-08T01:19:33,351 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=96982154c400,43235,1733620755742 2024-12-08T01:19:33,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T01:19:33,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 96982154c400%2C35801%2C1733620755391.meta.1733620757709.meta, worker=96982154c400,43235,1733620755742 in 265 msec 2024-12-08T01:19:33,355 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting dir is empty, no logs to split. 2024-12-08T01:19:33,355 INFO [PEWorker-1 {}] master.SplitWALManager(105): 96982154c400,35801,1733620755391 WAL count=0, meta=true 2024-12-08T01:19:33,355 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 96982154c400,35801,1733620755391 WAL splitting is done? wals=0, meta=true 2024-12-08T01:19:33,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T01:19:33,358 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:19:33,359 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T01:19:33,441 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:33,442 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1] 2024-12-08T01:19:33,443 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 96982154c400:35801 this server is in the failed servers list 2024-12-08T01:19:33,443 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=96982154c400:35801 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35801 2024-12-08T01:19:33,443 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35801 2024-12-08T01:19:33,443 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 from cache 2024-12-08T01:19:33,509 DEBUG [96982154c400:39725 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-08T01:19:33,510 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:19:33,510 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:19:33,510 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:19:33,510 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:19:33,510 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:19:33,510 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:19:33,510 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:19:33,510 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:19:33,510 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:33,512 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,37871,1733620755662, state=OPENING 2024-12-08T01:19:33,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:33,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:33,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:33,546 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T01:19:33,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:33,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:33,546 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:33,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,37871,1733620755662}] 2024-12-08T01:19:33,709 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T01:19:33,709 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:33,709 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T01:19:33,711 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=96982154c400%2C37871%2C1733620755662.meta, suffix=.meta, logDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662, archiveDir=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs, maxLogs=32 2024-12-08T01:19:33,725 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662/96982154c400%2C37871%2C1733620755662.meta.1733620773712.meta, exclude list is [], retry=0 2024-12-08T01:19:33,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:33,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:33,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:33,749 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,37871,1733620755662/96982154c400%2C37871%2C1733620755662.meta.1733620773712.meta 2024-12-08T01:19:33,749 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:33,749 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:33,750 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T01:19:33,750 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T01:19:33,750 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T01:19:33,750 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T01:19:33,750 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:33,750 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T01:19:33,750 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T01:19:33,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T01:19:33,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T01:19:33,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:33,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:33,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T01:19:33,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T01:19:33,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:33,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:33,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T01:19:33,755 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T01:19:33,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:33,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:33,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T01:19:33,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T01:19:33,756 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:33,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T01:19:33,757 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T01:19:33,757 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740 2024-12-08T01:19:33,759 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740 2024-12-08T01:19:33,760 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:19:33,785 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:33,787 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:19:33,787 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-08T01:19:33,804 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/info/5536c3e6b5954c8c9af61d99f0f2742b is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc./info:regioninfo/1733620772726/Put/seqid=0 2024-12-08T01:19:33,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741894_1072 (size=11177) 2024-12-08T01:19:33,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741894_1072 (size=11177) 2024-12-08T01:19:33,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741894_1072 (size=11177) 2024-12-08T01:19:33,811 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/info/5536c3e6b5954c8c9af61d99f0f2742b 2024-12-08T01:19:33,829 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/ns/2a24c9925027475aa5205d2d6acdb0d4 is 43, key is default/ns:d/1733620757882/Put/seqid=0 2024-12-08T01:19:33,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741895_1073 (size=5153) 2024-12-08T01:19:33,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741895_1073 (size=5153) 2024-12-08T01:19:33,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741895_1073 (size=5153) 2024-12-08T01:19:33,836 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/ns/2a24c9925027475aa5205d2d6acdb0d4 2024-12-08T01:19:33,856 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/table/afdda2e1233a4aaf8bbfbf1698e0e415 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733620771004/Put/seqid=0 2024-12-08T01:19:33,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741896_1074 (size=5431) 2024-12-08T01:19:33,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741896_1074 (size=5431) 2024-12-08T01:19:33,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741896_1074 (size=5431) 2024-12-08T01:19:33,863 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/table/afdda2e1233a4aaf8bbfbf1698e0e415 2024-12-08T01:19:33,869 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/info/5536c3e6b5954c8c9af61d99f0f2742b as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/info/5536c3e6b5954c8c9af61d99f0f2742b 2024-12-08T01:19:33,876 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/info/5536c3e6b5954c8c9af61d99f0f2742b, entries=36, sequenceid=18, filesize=10.9 K 2024-12-08T01:19:33,877 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/ns/2a24c9925027475aa5205d2d6acdb0d4 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/ns/2a24c9925027475aa5205d2d6acdb0d4 2024-12-08T01:19:33,884 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/ns/2a24c9925027475aa5205d2d6acdb0d4, entries=2, sequenceid=18, filesize=5.0 K 2024-12-08T01:19:33,885 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/table/afdda2e1233a4aaf8bbfbf1698e0e415 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/table/afdda2e1233a4aaf8bbfbf1698e0e415 2024-12-08T01:19:33,891 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/table/afdda2e1233a4aaf8bbfbf1698e0e415, entries=2, sequenceid=18, filesize=5.3 K 2024-12-08T01:19:33,892 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=18, compaction requested=false; wal=null 2024-12-08T01:19:33,892 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-08T01:19:33,894 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T01:19:33,894 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T01:19:33,895 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T01:19:33,896 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T01:19:33,898 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-08T01:19:33,899 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63046025, jitterRate=-0.06054101884365082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T01:19:33,899 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T01:19:33,900 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733620773750Writing region info on filesystem at 1733620773750Initializing all the Stores at 1733620773751 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620773751Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620773751Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620773751Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733620773751Obtaining lock to block concurrent updates at 1733620773787 (+36 ms)Preparing flush snapshotting stores in 1588230740 at 1733620773787Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733620773788 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733620773788Flushing 1588230740/info: creating writer at 1733620773788Flushing 1588230740/info: appending metadata at 1733620773803 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733620773803Flushing 1588230740/ns: creating writer at 1733620773816 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733620773829 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733620773829Flushing 1588230740/table: creating writer at 1733620773841 (+12 ms)Flushing 1588230740/table: appending metadata at 1733620773856 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733620773856Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@576ec628: reopening flushed file at 1733620773869 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53824ade: reopening flushed file at 1733620773876 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@212d51ec: reopening flushed file at 1733620773884 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=18, compaction requested=false; wal=null at 1733620773892 (+8 ms)Cleaning up temporary data from old regions at 1733620773894 (+2 ms)Running coprocessor post-open hooks at 1733620773899 (+5 ms)Region opened successfully at 1733620773900 (+1 ms) 2024-12-08T01:19:33,901 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733620773700 2024-12-08T01:19:33,903 DEBUG [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T01:19:33,903 INFO [RS_OPEN_META-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T01:19:33,904 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:33,904 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 96982154c400,37871,1733620755662, state=OPEN 2024-12-08T01:19:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T01:19:33,914 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=96982154c400,37871,1733620755662 2024-12-08T01:19:33,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:33,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:33,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T01:19:33,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-08T01:19:33,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=96982154c400,37871,1733620755662 in 368 msec 2024-12-08T01:19:33,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-08T01:19:33,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 560 msec 2024-12-08T01:19:33,920 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 96982154c400,35801,1733620755391 had 2 regions 2024-12-08T01:19:33,921 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 96982154c400,35801,1733620755391, splitWal=true, meta=true, isMeta: false 2024-12-08T01:19:33,923 INFO [PEWorker-2 {}] master.SplitWALManager(105): 96982154c400,35801,1733620755391 WAL count=1, meta=false 2024-12-08T01:19:33,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 96982154c400%2C35801%2C1733620755391.1733620757260}] 2024-12-08T01:19:33,925 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=96982154c400,37871,1733620755662 2024-12-08T01:19:33,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35801%2C1733620755391.1733620757260, worker=96982154c400,37871,1733620755662}] 2024-12-08T01:19:33,951 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:33,953 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,37871,1733620755662, seqNum=-1] 2024-12-08T01:19:34,072 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 96982154c400:35801 this server is in the failed servers list 2024-12-08T01:19:34,072 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=96982154c400:35801 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35801 2024-12-08T01:19:34,073 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35801 2024-12-08T01:19:34,073 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=12 from cache 2024-12-08T01:19:34,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37871 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-08T01:19:34,098 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260, size=2.2 K (2288bytes) 2024-12-08T01:19:34,098 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 2024-12-08T01:19:34,099 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 after 1ms 2024-12-08T01:19:34,120 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:34,120 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 took 22ms 2024-12-08T01:19:34,122 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 7b9fc8dd8c927db706d5818cc32f0abc: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-08T01:19:34,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 so closing down 2024-12-08T01:19:34,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:34,123 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:34,123 INFO [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-08T01:19:34,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260, journal: Splitting hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260, size=2.2 K (2288bytes) at 1733620774098Finishing writing output for hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 so closing down at 1733620774123 (+25 ms)3 split writer threads finished at 1733620774123Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733620774123 2024-12-08T01:19:34,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 2024-12-08T01:19:34,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/96982154c400:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-08T01:19:34,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39725 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-08T01:19:34,127 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting/96982154c400%2C35801%2C1733620755391.1733620757260 to hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs 2024-12-08T01:19:34,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-08T01:19:34,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 96982154c400%2C35801%2C1733620755391.1733620757260, worker=96982154c400,37871,1733620755662 in 201 msec 2024-12-08T01:19:34,131 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=96982154c400,37871,1733620755662 2024-12-08T01:19:34,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-08T01:19:34,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 96982154c400%2C35801%2C1733620755391.1733620757260, worker=96982154c400,37871,1733620755662 in 208 msec 2024-12-08T01:19:34,135 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/WALs/96982154c400,35801,1733620755391-splitting dir is empty, no logs to split. 2024-12-08T01:19:34,135 INFO [PEWorker-2 {}] master.SplitWALManager(105): 96982154c400,35801,1733620755391 WAL count=0, meta=false 2024-12-08T01:19:34,136 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 96982154c400,35801,1733620755391 WAL splitting is done? wals=0, meta=false 2024-12-08T01:19:34,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN}] 2024-12-08T01:19:34,139 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN 2024-12-08T01:19:34,140 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-08T01:19:34,290 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=18] 2024-12-08T01:19:34,290 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 96982154c400:35801 this server is in the failed servers list 2024-12-08T01:19:34,290 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=96982154c400:35801 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35801 2024-12-08T01:19:34,290 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(204): Hosts are {96982154c400=0} racks are {/default-rack=0} 2024-12-08T01:19:34,290 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 96982154c400:35801 2024-12-08T01:19:34,290 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,35801,1733620755391, seqNum=18 from cache 2024-12-08T01:19:34,291 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-08T01:19:34,291 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-08T01:19:34,291 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-08T01:19:34,291 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-08T01:19:34,291 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-08T01:19:34,291 INFO [96982154c400:39725 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-08T01:19:34,291 DEBUG [96982154c400:39725 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-08T01:19:34,291 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPENING, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:34,292 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 96982154c400:35801 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 96982154c400/172.17.0.2:35801 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:34,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1, error=java.net.ConnectException: Call to address=96982154c400:35801 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 96982154c400/172.17.0.2:35801 2024-12-08T01:19:34,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-08T01:19:34,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=96982154c400,35801,1733620755391, seqNum=-1 from cache 2024-12-08T01:19:34,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 96982154c400:35801 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 96982154c400/172.17.0.2:35801 2024-12-08T01:19:34,402 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T01:19:34,402 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=96982154c400,37871,1733620755662, seqNum=-1] 2024-12-08T01:19:34,403 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T01:19:34,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T01:19:34,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN because future has completed 2024-12-08T01:19:34,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662}] 2024-12-08T01:19:34,562 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:34,562 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b9fc8dd8c927db706d5818cc32f0abc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:34,563 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,563 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:34,563 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,563 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,565 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,566 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf1 2024-12-08T01:19:34,566 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,574 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf1/20020cb61c474860b3cd47882235a65e 2024-12-08T01:19:34,574 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,574 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,576 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b9fc8dd8c927db706d5818cc32f0abc columnFamilyName cf2 2024-12-08T01:19:34,576 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,583 DEBUG [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/cf2/bb70b41d5698418bab58658e035b2b04 2024-12-08T01:19:34,584 INFO [StoreOpener-7b9fc8dd8c927db706d5818cc32f0abc-1 {}] regionserver.HStore(327): Store=7b9fc8dd8c927db706d5818cc32f0abc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,584 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,585 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,586 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,587 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,587 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,588 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-08T01:19:34,589 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,590 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 7b9fc8dd8c927db706d5818cc32f0abc; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73768299, jitterRate=0.09923331439495087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-08T01:19:34,590 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:34,591 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 7b9fc8dd8c927db706d5818cc32f0abc: Running coprocessor pre-open hook at 1733620774563Writing region info on filesystem at 1733620774563Initializing all the Stores at 1733620774564 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774564Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774564Cleaning up temporary data from old regions at 1733620774587 (+23 ms)Running coprocessor post-open hooks at 1733620774590 (+3 ms)Region opened successfully at 1733620774591 (+1 ms) 2024-12-08T01:19:34,592 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., pid=21, masterSystemTime=1733620774559 2024-12-08T01:19:34,594 DEBUG [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:34,594 INFO [RS_OPEN_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:34,595 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7b9fc8dd8c927db706d5818cc32f0abc, regionState=OPEN, openSeqNum=18, regionLocation=96982154c400,37871,1733620755662 2024-12-08T01:19:34,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662 because future has completed 2024-12-08T01:19:34,601 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39725 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=96982154c400,37871,1733620755662, table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-08T01:19:34,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-08T01:19:34,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b9fc8dd8c927db706d5818cc32f0abc, server=96982154c400,37871,1733620755662 in 195 msec 2024-12-08T01:19:34,607 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., hostname=96982154c400,37871,1733620755662, seqNum=18] 2024-12-08T01:19:34,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-08T01:19:34,613 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 96982154c400,35801,1733620755391 after splitting done 2024-12-08T01:19:34,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=7b9fc8dd8c927db706d5818cc32f0abc, ASSIGN in 467 msec 2024-12-08T01:19:34,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 96982154c400,35801,1733620755391, splitWal=true, meta=true in 1.5400 sec 2024-12-08T01:19:34,622 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414 (was 417), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=9929 (was 9993) 2024-12-08T01:19:34,624 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-08T01:19:34,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=9927 2024-12-08T01:19:34,636 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-08T01:19:34,656 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:34,658 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:34,659 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:34,662 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98021071, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-98021071, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:34,677 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98021071/hregion-98021071.1733620774663, exclude list is [], retry=0 2024-12-08T01:19:34,680 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:34,680 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:34,681 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:34,704 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98021071/hregion-98021071.1733620774663 2024-12-08T01:19:34,705 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:34,705 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 428aeeee33b54dff59b2e0df3a3b581b, NAME => 'testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:34,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741898_1076 (size=67) 2024-12-08T01:19:34,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741898_1076 (size=67) 2024-12-08T01:19:34,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741898_1076 (size=67) 2024-12-08T01:19:34,716 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:34,717 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,720 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName a 2024-12-08T01:19:34,720 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,721 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,721 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,722 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName b 2024-12-08T01:19:34,722 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,723 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,723 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,724 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName c 2024-12-08T01:19:34,725 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,725 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,725 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,726 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,726 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,727 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,727 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,728 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:34,729 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,730 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:34,731 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 428aeeee33b54dff59b2e0df3a3b581b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69160915, jitterRate=0.030577942728996277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:34,731 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 428aeeee33b54dff59b2e0df3a3b581b: Writing region info on filesystem at 1733620774716Initializing all the Stores at 1733620774717 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774717Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774717Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774717Cleaning up temporary data from old regions at 1733620774727 (+10 ms)Region opened successfully at 1733620774731 (+4 ms) 2024-12-08T01:19:34,732 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 428aeeee33b54dff59b2e0df3a3b581b, disabling compactions & flushes 2024-12-08T01:19:34,732 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,732 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,732 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. after waiting 0 ms 2024-12-08T01:19:34,732 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,732 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,732 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 428aeeee33b54dff59b2e0df3a3b581b: Waiting for close lock at 1733620774731Disabling compacts and flushes for region at 1733620774731Disabling writes for close at 1733620774732 (+1 ms)Writing region close event to WAL at 1733620774732Closed at 1733620774732 2024-12-08T01:19:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741897_1075 (size=175) 2024-12-08T01:19:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741897_1075 (size=175) 2024-12-08T01:19:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741897_1075 (size=175) 2024-12-08T01:19:34,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:34,737 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98021071:(num 1733620774663) 2024-12-08T01:19:34,738 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:34,739 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:34,751 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740, exclude list is [], retry=0 2024-12-08T01:19:34,753 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:34,754 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:34,754 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:34,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 2024-12-08T01:19:34,777 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:34,777 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 428aeeee33b54dff59b2e0df3a3b581b, NAME => 'testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:34,777 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:34,777 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,778 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,779 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,779 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName a 2024-12-08T01:19:34,780 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,780 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,780 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,781 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName b 2024-12-08T01:19:34,781 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,781 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,781 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,782 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName c 2024-12-08T01:19:34,782 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:34,783 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:34,783 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,784 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,785 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,785 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,786 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:34,787 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:34,788 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 428aeeee33b54dff59b2e0df3a3b581b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66488234, jitterRate=-0.009248107671737671}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:34,788 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 428aeeee33b54dff59b2e0df3a3b581b: Writing region info on filesystem at 1733620774778Initializing all the Stores at 1733620774778Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774778Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774779 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620774779Cleaning up temporary data from old regions at 1733620774785 (+6 ms)Region opened successfully at 1733620774788 (+3 ms) 2024-12-08T01:19:34,815 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 428aeeee33b54dff59b2e0df3a3b581b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-08T01:19:34,830 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/a/9099c4fc75f84797a1c9c9dd345d02e5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733620774788/Put/seqid=0 2024-12-08T01:19:34,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741900_1078 (size=5958) 2024-12-08T01:19:34,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741900_1078 (size=5958) 2024-12-08T01:19:34,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741900_1078 (size=5958) 2024-12-08T01:19:34,837 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/a/9099c4fc75f84797a1c9c9dd345d02e5 2024-12-08T01:19:34,856 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/b/78c2cf74c00e408c96ebf8e5d1ecfdea is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733620774797/Put/seqid=0 2024-12-08T01:19:34,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741901_1079 (size=5958) 2024-12-08T01:19:34,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741901_1079 (size=5958) 2024-12-08T01:19:34,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741901_1079 (size=5958) 2024-12-08T01:19:34,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/b/78c2cf74c00e408c96ebf8e5d1ecfdea 2024-12-08T01:19:34,882 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/c/89650e0f41d74e6c91484eaf7249a7d9 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733620774807/Put/seqid=0 2024-12-08T01:19:34,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741902_1080 (size=5958) 2024-12-08T01:19:34,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741902_1080 (size=5958) 2024-12-08T01:19:34,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741902_1080 (size=5958) 2024-12-08T01:19:34,893 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/c/89650e0f41d74e6c91484eaf7249a7d9 2024-12-08T01:19:34,900 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/a/9099c4fc75f84797a1c9c9dd345d02e5 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/a/9099c4fc75f84797a1c9c9dd345d02e5 2024-12-08T01:19:34,906 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/a/9099c4fc75f84797a1c9c9dd345d02e5, entries=10, sequenceid=33, filesize=5.8 K 2024-12-08T01:19:34,907 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/b/78c2cf74c00e408c96ebf8e5d1ecfdea as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/b/78c2cf74c00e408c96ebf8e5d1ecfdea 2024-12-08T01:19:34,912 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/b/78c2cf74c00e408c96ebf8e5d1ecfdea, entries=10, sequenceid=33, filesize=5.8 K 2024-12-08T01:19:34,913 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/c/89650e0f41d74e6c91484eaf7249a7d9 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/c/89650e0f41d74e6c91484eaf7249a7d9 2024-12-08T01:19:34,918 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/c/89650e0f41d74e6c91484eaf7249a7d9, entries=10, sequenceid=33, filesize=5.8 K 2024-12-08T01:19:34,919 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 428aeeee33b54dff59b2e0df3a3b581b in 104ms, sequenceid=33, compaction requested=false 2024-12-08T01:19:34,919 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 428aeeee33b54dff59b2e0df3a3b581b: 2024-12-08T01:19:34,919 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 428aeeee33b54dff59b2e0df3a3b581b, disabling compactions & flushes 2024-12-08T01:19:34,919 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,920 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,920 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. after waiting 0 ms 2024-12-08T01:19:34,920 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,921 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. 2024-12-08T01:19:34,921 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 428aeeee33b54dff59b2e0df3a3b581b: Waiting for close lock at 1733620774919Disabling compacts and flushes for region at 1733620774919Disabling writes for close at 1733620774920 (+1 ms)Writing region close event to WAL at 1733620774921 (+1 ms)Closed at 1733620774921 2024-12-08T01:19:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741899_1077 (size=6311) 2024-12-08T01:19:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741899_1077 (size=6311) 2024-12-08T01:19:34,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741899_1077 (size=6311) 2024-12-08T01:19:34,928 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/b/78c2cf74c00e408c96ebf8e5d1ecfdea to hdfs://localhost:35931/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/b/78c2cf74c00e408c96ebf8e5d1ecfdea 2024-12-08T01:19:34,943 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740, size=6.2 K (6311bytes) 2024-12-08T01:19:34,943 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 2024-12-08T01:19:34,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 after 0ms 2024-12-08T01:19:34,971 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:34,971 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 took 28ms 2024-12-08T01:19:34,974 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 so closing down 2024-12-08T01:19:34,974 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:34,975 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620774740.temp 2024-12-08T01:19:34,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-08T01:19:34,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:34,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-08T01:19:34,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:34,998 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-08T01:19:34,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:34,998 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000003-wal.1733620774740.temp 2024-12-08T01:19:34,999 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:35,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741903_1081 (size=5575) 2024-12-08T01:19:35,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741903_1081 (size=5575) 2024-12-08T01:19:35,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741903_1081 (size=5575) 2024-12-08T01:19:35,007 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000003-wal.1733620774740.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:35,008 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000003-wal.1733620774740.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000032 2024-12-08T01:19:35,008 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-08T01:19:35,008 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740, journal: Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740, size=6.2 K (6311bytes) at 1733620774943Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 so closing down at 1733620774974 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000003-wal.1733620774740.temp at 1733620774998 (+24 ms)3 split writer threads finished at 1733620774999 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000003-wal.1733620774740.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733620775007 (+8 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000003-wal.1733620774740.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000032 at 1733620775008 (+1 ms)Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733620775008 2024-12-08T01:19:35,011 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620774740 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620774740 2024-12-08T01:19:35,011 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000032 2024-12-08T01:19:35,012 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:35,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:35,025 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620775013, exclude list is [], retry=0 2024-12-08T01:19:35,028 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:35,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:35,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:35,053 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620775013 2024-12-08T01:19:35,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:35,053 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 428aeeee33b54dff59b2e0df3a3b581b, NAME => 'testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:35,054 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:35,054 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,054 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,055 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,056 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName a 2024-12-08T01:19:35,056 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,062 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/a/9099c4fc75f84797a1c9c9dd345d02e5 2024-12-08T01:19:35,063 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,063 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,064 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName b 2024-12-08T01:19:35,065 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,065 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,065 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,066 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 428aeeee33b54dff59b2e0df3a3b581b columnFamilyName c 2024-12-08T01:19:35,066 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,072 DEBUG [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/c/89650e0f41d74e6c91484eaf7249a7d9 2024-12-08T01:19:35,072 INFO [StoreOpener-428aeeee33b54dff59b2e0df3a3b581b-1 {}] regionserver.HStore(327): Store=428aeeee33b54dff59b2e0df3a3b581b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,072 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,073 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,074 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,075 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000032 2024-12-08T01:19:35,097 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:35,099 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000032 2024-12-08T01:19:35,099 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 428aeeee33b54dff59b2e0df3a3b581b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-08T01:19:35,113 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/b/4c59bc76db57402c8668d83df3ad9205 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733620774797/Put/seqid=0 2024-12-08T01:19:35,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741905_1083 (size=5958) 2024-12-08T01:19:35,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741905_1083 (size=5958) 2024-12-08T01:19:35,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741905_1083 (size=5958) 2024-12-08T01:19:35,522 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/b/4c59bc76db57402c8668d83df3ad9205 2024-12-08T01:19:35,533 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/.tmp/b/4c59bc76db57402c8668d83df3ad9205 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/b/4c59bc76db57402c8668d83df3ad9205 2024-12-08T01:19:35,537 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/b/4c59bc76db57402c8668d83df3ad9205, entries=10, sequenceid=32, filesize=5.8 K 2024-12-08T01:19:35,538 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 428aeeee33b54dff59b2e0df3a3b581b in 439ms, sequenceid=32, compaction requested=false; wal=null 2024-12-08T01:19:35,538 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/0000000000000000032 2024-12-08T01:19:35,539 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,539 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,540 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:35,541 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 428aeeee33b54dff59b2e0df3a3b581b 2024-12-08T01:19:35,543 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/428aeeee33b54dff59b2e0df3a3b581b/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-08T01:19:35,544 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 428aeeee33b54dff59b2e0df3a3b581b; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74306831, jitterRate=0.10725806653499603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:35,545 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 428aeeee33b54dff59b2e0df3a3b581b: Writing region info on filesystem at 1733620775054Initializing all the Stores at 1733620775055 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775055Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775055Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775055Obtaining lock to block concurrent updates at 1733620775099 (+44 ms)Preparing flush snapshotting stores in 428aeeee33b54dff59b2e0df3a3b581b at 1733620775099Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733620775099Flushing stores of testReplayEditsWrittenViaHRegion,,1733620774657.428aeeee33b54dff59b2e0df3a3b581b. at 1733620775099Flushing 428aeeee33b54dff59b2e0df3a3b581b/b: creating writer at 1733620775099Flushing 428aeeee33b54dff59b2e0df3a3b581b/b: appending metadata at 1733620775113 (+14 ms)Flushing 428aeeee33b54dff59b2e0df3a3b581b/b: closing flushed file at 1733620775113Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5516c3e7: reopening flushed file at 1733620775531 (+418 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 428aeeee33b54dff59b2e0df3a3b581b in 439ms, sequenceid=32, compaction requested=false; wal=null at 1733620775538 (+7 ms)Cleaning up temporary data from old regions at 1733620775539 (+1 ms)Region opened successfully at 1733620775545 (+6 ms) 2024-12-08T01:19:35,566 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=422 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:51062 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:45972 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50874 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:50956 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:45800 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:51156 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1122 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 231), ProcessCount=11 (was 11), AvailableMemoryMB=9912 (was 9927) 2024-12-08T01:19:35,567 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-12-08T01:19:35,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=422, OpenFileDescriptor=1122, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=9912 2024-12-08T01:19:35,579 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-12-08T01:19:35,596 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:35,598 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:35,598 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:35,600 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54402933, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-54402933, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:35,610 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54402933/hregion-54402933.1733620775601, exclude list is [], retry=0 2024-12-08T01:19:35,613 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:35,613 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:35,613 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:35,635 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54402933/hregion-54402933.1733620775601 2024-12-08T01:19:35,636 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:35,636 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ac83f89b6b22c2f8435deaec37421aa6, NAME => 'testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:35,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741907_1085 (size=68) 2024-12-08T01:19:35,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741907_1085 (size=68) 2024-12-08T01:19:35,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741907_1085 (size=68) 2024-12-08T01:19:35,644 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:35,645 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,647 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName a 2024-12-08T01:19:35,647 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,647 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,647 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,648 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName b 2024-12-08T01:19:35,648 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,649 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,649 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,650 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName c 2024-12-08T01:19:35,650 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,650 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,650 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,652 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,652 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,653 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:35,654 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,656 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:35,656 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ac83f89b6b22c2f8435deaec37421aa6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58837870, jitterRate=-0.1232474148273468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:35,657 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ac83f89b6b22c2f8435deaec37421aa6: Writing region info on filesystem at 1733620775644Initializing all the Stores at 1733620775645 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775645Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775645Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775645Cleaning up temporary data from old regions at 1733620775652 (+7 ms)Region opened successfully at 1733620775657 (+5 ms) 2024-12-08T01:19:35,657 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ac83f89b6b22c2f8435deaec37421aa6, disabling compactions & flushes 2024-12-08T01:19:35,657 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:35,657 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:35,657 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. after waiting 0 ms 2024-12-08T01:19:35,657 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:35,657 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:35,657 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ac83f89b6b22c2f8435deaec37421aa6: Waiting for close lock at 1733620775657Disabling compacts and flushes for region at 1733620775657Disabling writes for close at 1733620775657Writing region close event to WAL at 1733620775657Closed at 1733620775657 2024-12-08T01:19:35,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741906_1084 (size=175) 2024-12-08T01:19:35,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741906_1084 (size=175) 2024-12-08T01:19:35,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741906_1084 (size=175) 2024-12-08T01:19:35,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:35,661 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54402933:(num 1733620775601) 2024-12-08T01:19:35,661 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:35,663 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:35,675 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663, exclude list is [], retry=0 2024-12-08T01:19:35,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:35,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:35,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:35,699 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 2024-12-08T01:19:35,700 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:35,747 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ac83f89b6b22c2f8435deaec37421aa6, NAME => 'testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:35,748 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,748 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:35,749 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,749 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,750 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,751 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName a 2024-12-08T01:19:35,751 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,752 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,752 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,753 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName b 2024-12-08T01:19:35,753 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,753 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,753 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,754 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName c 2024-12-08T01:19:35,754 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:35,754 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:35,754 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,757 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,757 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,758 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:35,759 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,759 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ac83f89b6b22c2f8435deaec37421aa6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69873662, jitterRate=0.04119870066642761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:35,759 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:35,760 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ac83f89b6b22c2f8435deaec37421aa6: Running coprocessor pre-open hook at 1733620775749Writing region info on filesystem at 1733620775749Initializing all the Stores at 1733620775750 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775750Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775750Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620775750Cleaning up temporary data from old regions at 1733620775757 (+7 ms)Running coprocessor post-open hooks at 1733620775759 (+2 ms)Region opened successfully at 1733620775760 (+1 ms) 2024-12-08T01:19:35,770 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ac83f89b6b22c2f8435deaec37421aa6 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-08T01:19:35,771 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:36,494 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:19:36,772 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:37,773 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:38,775 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:39,776 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:40,778 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:41,779 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:42,780 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:43,782 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:43,870 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T01:19:44,783 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ac83f89b6b22c2f8435deaec37421aa6/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:44,784 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ac83f89b6b22c2f8435deaec37421aa6: 2024-12-08T01:19:44,784 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:44,796 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ac83f89b6b22c2f8435deaec37421aa6: 2024-12-08T01:19:44,796 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-08T01:19:44,796 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ac83f89b6b22c2f8435deaec37421aa6, disabling compactions & flushes 2024-12-08T01:19:44,796 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:44,796 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:44,796 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. after waiting 0 ms 2024-12-08T01:19:44,797 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:44,797 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:44,797 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. 2024-12-08T01:19:44,797 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ac83f89b6b22c2f8435deaec37421aa6: Waiting for close lock at 1733620784796Running coprocessor pre-close hooks at 1733620784796Disabling compacts and flushes for region at 1733620784796Disabling writes for close at 1733620784797 (+1 ms)Writing region close event to WAL at 1733620784797Running coprocessor post-close hooks at 1733620784797Closed at 1733620784797 2024-12-08T01:19:44,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741908_1086 (size=4236) 2024-12-08T01:19:44,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741908_1086 (size=4236) 2024-12-08T01:19:44,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741908_1086 (size=4236) 2024-12-08T01:19:44,814 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663, size=4.1 K (4236bytes) 2024-12-08T01:19:44,814 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 2024-12-08T01:19:44,814 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 after 0ms 2024-12-08T01:19:44,838 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:44,838 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 took 24ms 2024-12-08T01:19:44,840 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 so closing down 2024-12-08T01:19:44,840 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:44,842 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733620775663.temp 2024-12-08T01:19:44,864 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000004-wal.1733620775663.temp 2024-12-08T01:19:44,865 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:44,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741909_1087 (size=3245) 2024-12-08T01:19:44,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741909_1087 (size=3245) 2024-12-08T01:19:44,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741909_1087 (size=3245) 2024-12-08T01:19:44,871 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000004-wal.1733620775663.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:44,872 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000004-wal.1733620775663.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000026 2024-12-08T01:19:44,872 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-08T01:19:44,872 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663, journal: Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663, size=4.1 K (4236bytes) at 1733620784814Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 so closing down at 1733620784840 (+26 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000004-wal.1733620775663.temp at 1733620784864 (+24 ms)3 split writer threads finished at 1733620784865 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000004-wal.1733620775663.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733620784871 (+6 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000004-wal.1733620775663.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000026 at 1733620784872 (+1 ms)Processed 23 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733620784872 2024-12-08T01:19:44,874 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620775663 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620775663 2024-12-08T01:19:44,875 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000026 2024-12-08T01:19:44,875 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:44,876 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:44,887 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620784876, exclude list is [], retry=0 2024-12-08T01:19:44,889 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:44,890 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:44,890 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:44,912 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620784876 2024-12-08T01:19:44,912 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:44,912 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ac83f89b6b22c2f8435deaec37421aa6, NAME => 'testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:44,913 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,913 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:44,913 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,913 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,915 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,916 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName a 2024-12-08T01:19:44,916 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:44,916 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:44,917 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,917 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName b 2024-12-08T01:19:44,917 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:44,918 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:44,918 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,918 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f89b6b22c2f8435deaec37421aa6 columnFamilyName c 2024-12-08T01:19:44,918 DEBUG [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:44,919 INFO [StoreOpener-ac83f89b6b22c2f8435deaec37421aa6-1 {}] regionserver.HStore(327): Store=ac83f89b6b22c2f8435deaec37421aa6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:44,919 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,920 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,921 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:44,922 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000026 2024-12-08T01:19:44,955 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:44,957 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000026 2024-12-08T01:19:44,957 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ac83f89b6b22c2f8435deaec37421aa6 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-08T01:19:44,970 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/a/02885ee3a0f04cc28b654b481fbe294e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733620784788/Put/seqid=0 2024-12-08T01:19:44,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741911_1089 (size=5523) 2024-12-08T01:19:44,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741911_1089 (size=5523) 2024-12-08T01:19:44,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741911_1089 (size=5523) 2024-12-08T01:19:44,978 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/a/02885ee3a0f04cc28b654b481fbe294e 2024-12-08T01:19:44,996 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-08T01:19:44,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:45,003 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/b/8d596bbe993c449395a545c56aa69e0e is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733620784784/Put/seqid=0 2024-12-08T01:19:45,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741912_1090 (size=5524) 2024-12-08T01:19:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741912_1090 (size=5524) 2024-12-08T01:19:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741912_1090 (size=5524) 2024-12-08T01:19:45,009 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/b/8d596bbe993c449395a545c56aa69e0e 2024-12-08T01:19:45,027 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/c/608bd9c57ffe4516839c27bd8c45e5c5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733620784786/Put/seqid=0 2024-12-08T01:19:45,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741913_1091 (size=5457) 2024-12-08T01:19:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741913_1091 (size=5457) 2024-12-08T01:19:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741913_1091 (size=5457) 2024-12-08T01:19:45,034 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/c/608bd9c57ffe4516839c27bd8c45e5c5 2024-12-08T01:19:45,041 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/a/02885ee3a0f04cc28b654b481fbe294e as hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/a/02885ee3a0f04cc28b654b481fbe294e 2024-12-08T01:19:45,047 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/a/02885ee3a0f04cc28b654b481fbe294e, entries=7, sequenceid=26, filesize=5.4 K 2024-12-08T01:19:45,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/b/8d596bbe993c449395a545c56aa69e0e as hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/b/8d596bbe993c449395a545c56aa69e0e 2024-12-08T01:19:45,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/b/8d596bbe993c449395a545c56aa69e0e, entries=7, sequenceid=26, filesize=5.4 K 2024-12-08T01:19:45,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/.tmp/c/608bd9c57ffe4516839c27bd8c45e5c5 as hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/c/608bd9c57ffe4516839c27bd8c45e5c5 2024-12-08T01:19:45,061 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/c/608bd9c57ffe4516839c27bd8c45e5c5, entries=6, sequenceid=26, filesize=5.3 K 2024-12-08T01:19:45,061 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ac83f89b6b22c2f8435deaec37421aa6 in 104ms, sequenceid=26, compaction requested=false; wal=null 2024-12-08T01:19:45,062 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/0000000000000000026 2024-12-08T01:19:45,063 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:45,063 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:45,064 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:45,065 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:45,068 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsAfterAbortingFlush/ac83f89b6b22c2f8435deaec37421aa6/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-08T01:19:45,069 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ac83f89b6b22c2f8435deaec37421aa6; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73434515, jitterRate=0.09425954520702362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:45,069 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac83f89b6b22c2f8435deaec37421aa6 2024-12-08T01:19:45,070 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ac83f89b6b22c2f8435deaec37421aa6: Running coprocessor pre-open hook at 1733620784913Writing region info on filesystem at 1733620784913Initializing all the Stores at 1733620784914 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620784914Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620784914Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620784914Obtaining lock to block concurrent updates at 1733620784957 (+43 ms)Preparing flush snapshotting stores in ac83f89b6b22c2f8435deaec37421aa6 at 1733620784957Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733620784957Flushing stores of testReplayEditsAfterAbortingFlush,,1733620775597.ac83f89b6b22c2f8435deaec37421aa6. at 1733620784957Flushing ac83f89b6b22c2f8435deaec37421aa6/a: creating writer at 1733620784958 (+1 ms)Flushing ac83f89b6b22c2f8435deaec37421aa6/a: appending metadata at 1733620784970 (+12 ms)Flushing ac83f89b6b22c2f8435deaec37421aa6/a: closing flushed file at 1733620784970Flushing ac83f89b6b22c2f8435deaec37421aa6/b: creating writer at 1733620784984 (+14 ms)Flushing ac83f89b6b22c2f8435deaec37421aa6/b: appending metadata at 1733620785002 (+18 ms)Flushing ac83f89b6b22c2f8435deaec37421aa6/b: closing flushed file at 1733620785002Flushing ac83f89b6b22c2f8435deaec37421aa6/c: creating writer at 1733620785014 (+12 ms)Flushing ac83f89b6b22c2f8435deaec37421aa6/c: appending metadata at 1733620785026 (+12 ms)Flushing ac83f89b6b22c2f8435deaec37421aa6/c: closing flushed file at 1733620785026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f2544de: reopening flushed file at 1733620785040 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e9c3919: reopening flushed file at 1733620785047 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53f8a5cf: reopening flushed file at 1733620785054 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ac83f89b6b22c2f8435deaec37421aa6 in 104ms, sequenceid=26, compaction requested=false; wal=null at 1733620785061 (+7 ms)Cleaning up temporary data from old regions at 1733620785063 (+2 ms)Running coprocessor post-open hooks at 1733620785069 (+6 ms)Region opened successfully at 1733620785069 2024-12-08T01:19:45,088 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417 (was 422), OpenFileDescriptor=1178 (was 1122) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=195 (was 231), ProcessCount=11 (was 11), AvailableMemoryMB=9856 (was 9912) 2024-12-08T01:19:45,088 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-12-08T01:19:45,099 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1178, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=9855 2024-12-08T01:19:45,099 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-12-08T01:19:45,112 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:45,133 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:45,134 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:45,137 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45782110, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-45782110, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:45,149 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45782110/hregion-45782110.1733620785137, exclude list is [], retry=0 2024-12-08T01:19:45,151 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:45,152 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:45,153 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:45,189 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45782110/hregion-45782110.1733620785137 2024-12-08T01:19:45,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:45,190 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 40dadff890f25a3d61e1e9b7cababede, NAME => 'testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:45,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741915_1093 (size=61) 2024-12-08T01:19:45,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741915_1093 (size=61) 2024-12-08T01:19:45,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741915_1093 (size=61) 2024-12-08T01:19:45,199 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:45,200 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,202 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40dadff890f25a3d61e1e9b7cababede columnFamilyName a 2024-12-08T01:19:45,202 DEBUG [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,202 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(327): Store=40dadff890f25a3d61e1e9b7cababede/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,203 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,204 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,204 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,205 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,207 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:45,207 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40dadff890f25a3d61e1e9b7cababede; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68138276, jitterRate=0.015339434146881104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40dadff890f25a3d61e1e9b7cababede: Writing region info on filesystem at 1733620785199Initializing all the Stores at 1733620785199Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785200 (+1 ms)Cleaning up temporary data from old regions at 1733620785204 (+4 ms)Region opened successfully at 1733620785208 (+4 ms) 2024-12-08T01:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40dadff890f25a3d61e1e9b7cababede, disabling compactions & flushes 2024-12-08T01:19:45,208 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. after waiting 0 ms 2024-12-08T01:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,208 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40dadff890f25a3d61e1e9b7cababede: Waiting for close lock at 1733620785208Disabling compacts and flushes for region at 1733620785208Disabling writes for close at 1733620785208Writing region close event to WAL at 1733620785208Closed at 1733620785208 2024-12-08T01:19:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741914_1092 (size=175) 2024-12-08T01:19:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741914_1092 (size=175) 2024-12-08T01:19:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741914_1092 (size=175) 2024-12-08T01:19:45,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:45,216 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45782110:(num 1733620785137) 2024-12-08T01:19:45,216 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:45,218 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:45,229 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218, exclude list is [], retry=0 2024-12-08T01:19:45,231 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:45,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:45,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:45,255 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 2024-12-08T01:19:45,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:45,256 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40dadff890f25a3d61e1e9b7cababede, NAME => 'testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:45,256 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:45,256 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,256 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,257 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,258 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40dadff890f25a3d61e1e9b7cababede columnFamilyName a 2024-12-08T01:19:45,258 DEBUG [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,259 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(327): Store=40dadff890f25a3d61e1e9b7cababede/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,259 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,260 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,261 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,261 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,262 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,263 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40dadff890f25a3d61e1e9b7cababede; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59462723, jitterRate=-0.11393637955188751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:45,263 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40dadff890f25a3d61e1e9b7cababede: Writing region info on filesystem at 1733620785256Initializing all the Stores at 1733620785257 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785257Cleaning up temporary data from old regions at 1733620785261 (+4 ms)Region opened successfully at 1733620785263 (+2 ms) 2024-12-08T01:19:45,270 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40dadff890f25a3d61e1e9b7cababede, disabling compactions & flushes 2024-12-08T01:19:45,270 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,270 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,270 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. after waiting 0 ms 2024-12-08T01:19:45,270 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,271 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,271 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. 2024-12-08T01:19:45,271 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40dadff890f25a3d61e1e9b7cababede: Waiting for close lock at 1733620785270Disabling compacts and flushes for region at 1733620785270Disabling writes for close at 1733620785270Writing region close event to WAL at 1733620785271 (+1 ms)Closed at 1733620785271 2024-12-08T01:19:45,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741916_1094 (size=1795) 2024-12-08T01:19:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741916_1094 (size=1795) 2024-12-08T01:19:45,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741916_1094 (size=1795) 2024-12-08T01:19:45,288 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218, size=1.8 K (1795bytes) 2024-12-08T01:19:45,288 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 2024-12-08T01:19:45,288 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 after 0ms 2024-12-08T01:19:45,312 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:45,312 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 took 24ms 2024-12-08T01:19:45,314 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 so closing down 2024-12-08T01:19:45,314 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:45,315 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620785218.temp 2024-12-08T01:19:45,339 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000003-wal.1733620785218.temp 2024-12-08T01:19:45,340 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741917_1095 (size=1795) 2024-12-08T01:19:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741917_1095 (size=1795) 2024-12-08T01:19:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741917_1095 (size=1795) 2024-12-08T01:19:45,346 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000003-wal.1733620785218.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:45,348 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000003-wal.1733620785218.temp to hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000012 2024-12-08T01:19:45,348 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-08T01:19:45,348 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218, journal: Splitting hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218, size=1.8 K (1795bytes) at 1733620785288Finishing writing output for hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 so closing down at 1733620785314 (+26 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000003-wal.1733620785218.temp at 1733620785339 (+25 ms)3 split writer threads finished at 1733620785340 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000003-wal.1733620785218.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733620785346 (+6 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000003-wal.1733620785218.temp to hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000012 at 1733620785348 (+2 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733620785348 2024-12-08T01:19:45,350 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785218 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620785218 2024-12-08T01:19:45,350 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000012 2024-12-08T01:19:45,384 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:45,712 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:45,714 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:45,725 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785715, exclude list is [], retry=0 2024-12-08T01:19:45,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:45,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:45,728 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:45,749 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785715 2024-12-08T01:19:45,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:45,750 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40dadff890f25a3d61e1e9b7cababede, NAME => 'testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:45,750 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:45,750 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,750 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,753 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,754 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40dadff890f25a3d61e1e9b7cababede columnFamilyName a 2024-12-08T01:19:45,754 DEBUG [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,755 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(327): Store=40dadff890f25a3d61e1e9b7cababede/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,755 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,758 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000012 2024-12-08T01:19:45,781 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:45,782 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000012 2024-12-08T01:19:45,782 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 40dadff890f25a3d61e1e9b7cababede 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-08T01:19:45,796 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/.tmp/a/c5726f3fc94f4d768ec68842e5bab48d is 79, key is testDatalossWhenInputError/a:x0/1733620785263/Put/seqid=0 2024-12-08T01:19:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741919_1097 (size=5808) 2024-12-08T01:19:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741919_1097 (size=5808) 2024-12-08T01:19:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741919_1097 (size=5808) 2024-12-08T01:19:45,803 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/.tmp/a/c5726f3fc94f4d768ec68842e5bab48d 2024-12-08T01:19:45,811 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/.tmp/a/c5726f3fc94f4d768ec68842e5bab48d as hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/a/c5726f3fc94f4d768ec68842e5bab48d 2024-12-08T01:19:45,820 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/a/c5726f3fc94f4d768ec68842e5bab48d, entries=10, sequenceid=12, filesize=5.7 K 2024-12-08T01:19:45,820 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 40dadff890f25a3d61e1e9b7cababede in 38ms, sequenceid=12, compaction requested=false; wal=null 2024-12-08T01:19:45,821 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/0000000000000000012 2024-12-08T01:19:45,822 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,822 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,825 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,827 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-08T01:19:45,828 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40dadff890f25a3d61e1e9b7cababede; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73883148, jitterRate=0.10094469785690308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:45,829 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40dadff890f25a3d61e1e9b7cababede: Writing region info on filesystem at 1733620785750Initializing all the Stores at 1733620785753 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785753Obtaining lock to block concurrent updates at 1733620785782 (+29 ms)Preparing flush snapshotting stores in 40dadff890f25a3d61e1e9b7cababede at 1733620785782Finished memstore snapshotting testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733620785782Flushing stores of testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede. at 1733620785782Flushing 40dadff890f25a3d61e1e9b7cababede/a: creating writer at 1733620785782Flushing 40dadff890f25a3d61e1e9b7cababede/a: appending metadata at 1733620785796 (+14 ms)Flushing 40dadff890f25a3d61e1e9b7cababede/a: closing flushed file at 1733620785796Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2011ca7b: reopening flushed file at 1733620785809 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 40dadff890f25a3d61e1e9b7cababede in 38ms, sequenceid=12, compaction requested=false; wal=null at 1733620785820 (+11 ms)Cleaning up temporary data from old regions at 1733620785822 (+2 ms)Region opened successfully at 1733620785829 (+7 ms) 2024-12-08T01:19:45,832 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40dadff890f25a3d61e1e9b7cababede, NAME => 'testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:45,832 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733620785112.40dadff890f25a3d61e1e9b7cababede.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:45,832 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,832 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,834 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,835 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40dadff890f25a3d61e1e9b7cababede columnFamilyName a 2024-12-08T01:19:45,835 DEBUG [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,841 DEBUG [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/a/c5726f3fc94f4d768ec68842e5bab48d 2024-12-08T01:19:45,841 INFO [StoreOpener-40dadff890f25a3d61e1e9b7cababede-1 {}] regionserver.HStore(327): Store=40dadff890f25a3d61e1e9b7cababede/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,842 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,844 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,844 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,844 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,847 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40dadff890f25a3d61e1e9b7cababede 2024-12-08T01:19:45,850 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testDatalossWhenInputError/40dadff890f25a3d61e1e9b7cababede/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-08T01:19:45,851 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40dadff890f25a3d61e1e9b7cababede; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74234191, jitterRate=0.10617564618587494}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-08T01:19:45,851 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40dadff890f25a3d61e1e9b7cababede: Writing region info on filesystem at 1733620785832Initializing all the Stores at 1733620785833 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785833Cleaning up temporary data from old regions at 1733620785844 (+11 ms)Region opened successfully at 1733620785851 (+7 ms) 2024-12-08T01:19:45,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=428 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:49752 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:49368 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:49808 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:49276 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:58252 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:58262 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1381908131-172.17.0.2-1733620750211:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1178) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=195 (was 195), ProcessCount=11 (was 11), AvailableMemoryMB=9811 (was 9855) 2024-12-08T01:19:45,869 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-12-08T01:19:45,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=9810 2024-12-08T01:19:45,880 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-12-08T01:19:45,893 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:45,895 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:45,896 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:45,898 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-42094605, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-42094605, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:45,909 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-42094605/hregion-42094605.1733620785898, exclude list is [], retry=0 2024-12-08T01:19:45,911 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:45,911 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:45,912 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:45,934 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-42094605/hregion-42094605.1733620785898 2024-12-08T01:19:45,935 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:45,935 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c19baaee4790031f8d1d87a130b38825, NAME => 'testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:45,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741921_1099 (size=63) 2024-12-08T01:19:45,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741921_1099 (size=63) 2024-12-08T01:19:45,943 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741921_1099 (size=63) 2024-12-08T01:19:45,944 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,945 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName a 2024-12-08T01:19:45,945 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,946 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,946 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,947 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName b 2024-12-08T01:19:45,947 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,947 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,947 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,949 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName c 2024-12-08T01:19:45,949 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:45,949 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:45,949 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,952 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:45,956 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:45,958 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:45,959 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c19baaee4790031f8d1d87a130b38825; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64947951, jitterRate=-0.03220011293888092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:45,959 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c19baaee4790031f8d1d87a130b38825: Writing region info on filesystem at 1733620785943Initializing all the Stores at 1733620785944 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785944Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785944Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620785944Cleaning up temporary data from old regions at 1733620785951 (+7 ms)Region opened successfully at 1733620785959 (+8 ms) 2024-12-08T01:19:45,959 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c19baaee4790031f8d1d87a130b38825, disabling compactions & flushes 2024-12-08T01:19:45,959 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:45,959 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:45,959 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. after waiting 0 ms 2024-12-08T01:19:45,959 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:45,960 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:45,960 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c19baaee4790031f8d1d87a130b38825: Waiting for close lock at 1733620785959Disabling compacts and flushes for region at 1733620785959Disabling writes for close at 1733620785959Writing region close event to WAL at 1733620785960 (+1 ms)Closed at 1733620785960 2024-12-08T01:19:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741920_1098 (size=175) 2024-12-08T01:19:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741920_1098 (size=175) 2024-12-08T01:19:45,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741920_1098 (size=175) 2024-12-08T01:19:45,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:45,965 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-42094605:(num 1733620785898) 2024-12-08T01:19:45,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:45,967 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:45,979 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967, exclude list is [], retry=0 2024-12-08T01:19:45,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:45,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:45,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:46,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 2024-12-08T01:19:46,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:46,002 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c19baaee4790031f8d1d87a130b38825, NAME => 'testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:46,002 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:46,002 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,002 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,004 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,004 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName a 2024-12-08T01:19:46,004 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:46,005 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:46,005 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,006 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName b 2024-12-08T01:19:46,006 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:46,006 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:46,006 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,007 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName c 2024-12-08T01:19:46,007 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:46,007 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:46,007 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,010 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,010 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,010 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:46,011 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:46,012 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c19baaee4790031f8d1d87a130b38825; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63188801, jitterRate=-0.058413490653038025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:46,012 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c19baaee4790031f8d1d87a130b38825: Writing region info on filesystem at 1733620786002Initializing all the Stores at 1733620786003 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620786003Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620786003Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620786003Cleaning up temporary data from old regions at 1733620786010 (+7 ms)Region opened successfully at 1733620786012 (+2 ms) 2024-12-08T01:19:46,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733620786015/Put/seqid=0 2024-12-08T01:19:46,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741923_1101 (size=4875) 2024-12-08T01:19:46,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741923_1101 (size=4875) 2024-12-08T01:19:46,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741923_1101 (size=4875) 2024-12-08T01:19:46,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733620786025/Put/seqid=0 2024-12-08T01:19:46,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741924_1102 (size=4875) 2024-12-08T01:19:46,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741924_1102 (size=4875) 2024-12-08T01:19:46,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741924_1102 (size=4875) 2024-12-08T01:19:46,037 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733620786037/Put/seqid=0 2024-12-08T01:19:46,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741925_1103 (size=4875) 2024-12-08T01:19:46,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741925_1103 (size=4875) 2024-12-08T01:19:46,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741925_1103 (size=4875) 2024-12-08T01:19:46,045 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c19baaee4790031f8d1d87a130b38825/a 2024-12-08T01:19:46,048 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-08T01:19:46,048 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:19:46,048 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c19baaee4790031f8d1d87a130b38825/a 2024-12-08T01:19:46,052 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-08T01:19:46,052 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:19:46,052 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c19baaee4790031f8d1d87a130b38825/a 2024-12-08T01:19:46,055 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-08T01:19:46,055 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-08T01:19:46,055 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c19baaee4790031f8d1d87a130b38825 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-08T01:19:46,076 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/.tmp/a/b212a1e0c7524f3999b8562989a71d41 is 55, key is testCompactedBulkLoadedFiles/a:a/1733620786013/Put/seqid=0 2024-12-08T01:19:46,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741926_1104 (size=5107) 2024-12-08T01:19:46,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741926_1104 (size=5107) 2024-12-08T01:19:46,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741926_1104 (size=5107) 2024-12-08T01:19:46,097 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/.tmp/a/b212a1e0c7524f3999b8562989a71d41 2024-12-08T01:19:46,102 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/.tmp/a/b212a1e0c7524f3999b8562989a71d41 as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41 2024-12-08T01:19:46,107 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41, entries=1, sequenceid=4, filesize=5.0 K 2024-12-08T01:19:46,108 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c19baaee4790031f8d1d87a130b38825 in 53ms, sequenceid=4, compaction requested=false 2024-12-08T01:19:46,108 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c19baaee4790031f8d1d87a130b38825: 2024-12-08T01:19:46,109 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ 2024-12-08T01:19:46,110 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ 2024-12-08T01:19:46,111 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ 2024-12-08T01:19:46,112 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile0 into c19baaee4790031f8d1d87a130b38825/a as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ - updating store file list. 2024-12-08T01:19:46,116 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b8305f02562746f3963592cba223568f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:46,116 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ into c19baaee4790031f8d1d87a130b38825/a 2024-12-08T01:19:46,116 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile0 into c19baaee4790031f8d1d87a130b38825/a (new location: hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_) 2024-12-08T01:19:46,118 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile1 into c19baaee4790031f8d1d87a130b38825/a as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ - updating store file list. 2024-12-08T01:19:46,122 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:46,122 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ into c19baaee4790031f8d1d87a130b38825/a 2024-12-08T01:19:46,122 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile1 into c19baaee4790031f8d1d87a130b38825/a (new location: hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_) 2024-12-08T01:19:46,124 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile2 into c19baaee4790031f8d1d87a130b38825/a as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ - updating store file list. 2024-12-08T01:19:46,128 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:46,128 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ into c19baaee4790031f8d1d87a130b38825/a 2024-12-08T01:19:46,128 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35931/hbase/testCompactedBulkLoadedFiles/hfile2 into c19baaee4790031f8d1d87a130b38825/a (new location: hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_) 2024-12-08T01:19:46,135 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T01:19:46,135 DEBUG [Time-limited test {}] regionserver.HStore(1541): c19baaee4790031f8d1d87a130b38825/a is initiating major compaction (all files) 2024-12-08T01:19:46,135 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c19baaee4790031f8d1d87a130b38825/a in testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:46,135 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_] into tmpdir=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/.tmp, totalSize=19.3 K 2024-12-08T01:19:46,136 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b212a1e0c7524f3999b8562989a71d41, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733620786013 2024-12-08T01:19:46,136 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b8305f02562746f3963592cba223568f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-08T01:19:46,137 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-08T01:19:46,137 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-08T01:19:46,148 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/.tmp/a/fbe978454537415dbda659a6a1a7fe87 is 55, key is testCompactedBulkLoadedFiles/a:a/1733620786013/Put/seqid=0 2024-12-08T01:19:46,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741927_1105 (size=6154) 2024-12-08T01:19:46,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741927_1105 (size=6154) 2024-12-08T01:19:46,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741927_1105 (size=6154) 2024-12-08T01:19:46,160 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/.tmp/a/fbe978454537415dbda659a6a1a7fe87 as hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/fbe978454537415dbda659a6a1a7fe87 2024-12-08T01:19:46,165 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c19baaee4790031f8d1d87a130b38825/a of c19baaee4790031f8d1d87a130b38825 into fbe978454537415dbda659a6a1a7fe87(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T01:19:46,165 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c19baaee4790031f8d1d87a130b38825: 2024-12-08T01:19:46,165 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-08T01:19:46,165 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-08T01:19:46,193 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967, size=0 (0bytes) 2024-12-08T01:19:46,193 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 might be still open, length is 0 2024-12-08T01:19:46,193 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 2024-12-08T01:19:46,193 WARN [IPC Server handler 2 on default port 35931 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-08T01:19:46,193 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 after 0ms 2024-12-08T01:19:46,822 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:49846 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39611:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49846 dst: /127.0.0.1:39611 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39611 remote=/127.0.0.1:49846]. Total timeout mills is 60000, 59343 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:46,822 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:49410 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49410 dst: /127.0.0.1:41533 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:46,822 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:58292 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58292 dst: /127.0.0.1:39867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:46,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741922_1106 (size=1794) 2024-12-08T01:19:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741922_1106 (size=1794) 2024-12-08T01:19:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741922_1106 (size=1794) 2024-12-08T01:19:50,194 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 after 4001ms 2024-12-08T01:19:50,225 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:50,225 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 took 4033ms 2024-12-08T01:19:50,227 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967; continuing. 2024-12-08T01:19:50,227 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 so closing down 2024-12-08T01:19:50,227 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:50,228 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620785967.temp 2024-12-08T01:19:50,251 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000003-wal.1733620785967.temp 2024-12-08T01:19:50,252 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741928_1107 (size=800) 2024-12-08T01:19:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741928_1107 (size=800) 2024-12-08T01:19:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741928_1107 (size=800) 2024-12-08T01:19:50,258 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000003-wal.1733620785967.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:50,259 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000003-wal.1733620785967.temp to hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000008 2024-12-08T01:19:50,259 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:19:50,259 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967, journal: Splitting hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967, size=0 (0bytes) at 1733620786193Finishing writing output for hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 so closing down at 1733620790227 (+4034 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000003-wal.1733620785967.temp at 1733620790251 (+24 ms)3 split writer threads finished at 1733620790252 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000003-wal.1733620785967.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733620790258 (+6 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000003-wal.1733620785967.temp to hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000008 at 1733620790259 (+1 ms)Processed 5 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967, size=0, length=0, corrupted=false, cancelled=false at 1733620790259 2024-12-08T01:19:50,260 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620785967 2024-12-08T01:19:50,261 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000008 2024-12-08T01:19:50,261 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:50,263 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:50,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620790263, exclude list is [], retry=0 2024-12-08T01:19:50,275 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:50,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:50,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:50,297 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620790263 2024-12-08T01:19:50,297 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:50,297 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c19baaee4790031f8d1d87a130b38825, NAME => 'testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:50,297 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:50,298 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,298 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,299 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,300 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName a 2024-12-08T01:19:50,300 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,306 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41 2024-12-08T01:19:50,309 DEBUG [StoreFileOpener-c19baaee4790031f8d1d87a130b38825-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b8305f02562746f3963592cba223568f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:50,309 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ 2024-12-08T01:19:50,313 DEBUG [StoreFileOpener-c19baaee4790031f8d1d87a130b38825-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:50,313 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ 2024-12-08T01:19:50,315 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T01:19:50,317 DEBUG [StoreFileOpener-c19baaee4790031f8d1d87a130b38825-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-08T01:19:50,317 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ 2024-12-08T01:19:50,322 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/fbe978454537415dbda659a6a1a7fe87 2024-12-08T01:19:50,322 WARN [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-12-08T01:19:50,322 WARN [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-12-08T01:19:50,322 WARN [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-12-08T01:19:50,322 WARN [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6fd9e869 2024-12-08T01:19:50,322 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_] to archive 2024-12-08T01:19:50,323 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T01:19:50,325 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41 to hdfs://localhost:35931/hbase/archive/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b212a1e0c7524f3999b8562989a71d41 2024-12-08T01:19:50,326 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ to hdfs://localhost:35931/hbase/archive/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/b8305f02562746f3963592cba223568f_SeqId_4_ 2024-12-08T01:19:50,327 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ to hdfs://localhost:35931/hbase/archive/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_ 2024-12-08T01:19:50,328 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ to hdfs://localhost:35931/hbase/archive/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/a/e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_ 2024-12-08T01:19:50,328 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,328 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,329 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName b 2024-12-08T01:19:50,329 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,329 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,329 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,330 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c19baaee4790031f8d1d87a130b38825 columnFamilyName c 2024-12-08T01:19:50,330 DEBUG [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,331 INFO [StoreOpener-c19baaee4790031f8d1d87a130b38825-1 {}] regionserver.HStore(327): Store=c19baaee4790031f8d1d87a130b38825/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,331 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,333 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,334 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000008 2024-12-08T01:19:50,370 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:50,372 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c19baaee4790031f8d1d87a130b38825 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c19baaee4790031f8d1d87a130b38825" family_name: "a" compaction_input: "b212a1e0c7524f3999b8562989a71d41" compaction_input: "b8305f02562746f3963592cba223568f_SeqId_4_" compaction_input: "c208bd2466bc4e9fbf0f8d1632685826_SeqId_4_" compaction_input: "e5a6ad838da94ce1b2ed9f4eff49968b_SeqId_4_" compaction_output: "fbe978454537415dbda659a6a1a7fe87" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-08T01:19:50,372 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-08T01:19:50,372 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000008 2024-12-08T01:19:50,373 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/0000000000000000008 2024-12-08T01:19:50,374 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,374 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,375 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:50,376 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c19baaee4790031f8d1d87a130b38825 2024-12-08T01:19:50,378 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testCompactedBulkLoadedFiles/c19baaee4790031f8d1d87a130b38825/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-08T01:19:50,379 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c19baaee4790031f8d1d87a130b38825; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74985126, jitterRate=0.11736544966697693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:50,379 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c19baaee4790031f8d1d87a130b38825: Writing region info on filesystem at 1733620790298Initializing all the Stores at 1733620790299 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790299Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790299Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790299Cleaning up temporary data from old regions at 1733620790374 (+75 ms)Region opened successfully at 1733620790379 (+5 ms) 2024-12-08T01:19:50,381 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c19baaee4790031f8d1d87a130b38825, disabling compactions & flushes 2024-12-08T01:19:50,382 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. after waiting 0 ms 2024-12-08T01:19:50,382 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:50,383 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733620785894.c19baaee4790031f8d1d87a130b38825. 2024-12-08T01:19:50,383 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c19baaee4790031f8d1d87a130b38825: Waiting for close lock at 1733620790381Disabling compacts and flushes for region at 1733620790381Disabling writes for close at 1733620790382 (+1 ms)Writing region close event to WAL at 1733620790382Closed at 1733620790382 2024-12-08T01:19:50,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741929_1108 (size=175) 2024-12-08T01:19:50,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741929_1108 (size=175) 2024-12-08T01:19:50,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741929_1108 (size=175) 2024-12-08T01:19:50,388 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:50,388 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620790263) 2024-12-08T01:19:50,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=441 (was 428) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1701031847_22 at /127.0.0.1:38150 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35931 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35931 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1701031847_22 at /127.0.0.1:41754 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1345 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 195), ProcessCount=11 (was 11), AvailableMemoryMB=9788 (was 9810) 2024-12-08T01:19:50,401 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1345 is superior to 1024 2024-12-08T01:19:50,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=441, OpenFileDescriptor=1345, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=9788 2024-12-08T01:19:50,412 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1345 is superior to 1024 2024-12-08T01:19:50,424 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:50,425 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T01:19:50,426 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T01:19:50,429 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24241128, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/hregion-24241128, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:50,441 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24241128/hregion-24241128.1733620790429, exclude list is [], retry=0 2024-12-08T01:19:50,444 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:50,444 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:50,444 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:50,480 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24241128/hregion-24241128.1733620790429 2024-12-08T01:19:50,481 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:50,481 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9d41be4f95858ff5b8c90ef34d6d4282, NAME => 'testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35931/hbase 2024-12-08T01:19:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741931_1110 (size=67) 2024-12-08T01:19:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741931_1110 (size=67) 2024-12-08T01:19:50,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741931_1110 (size=67) 2024-12-08T01:19:50,489 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:50,490 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,491 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName a 2024-12-08T01:19:50,491 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,492 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,492 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,493 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName b 2024-12-08T01:19:50,493 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,493 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,493 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,494 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName c 2024-12-08T01:19:50,494 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,494 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,496 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,496 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,497 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:50,498 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,499 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T01:19:50,500 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d41be4f95858ff5b8c90ef34d6d4282; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63225857, jitterRate=-0.057861313223838806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:50,500 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d41be4f95858ff5b8c90ef34d6d4282: Writing region info on filesystem at 1733620790489Initializing all the Stores at 1733620790490 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790490Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790490Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790490Cleaning up temporary data from old regions at 1733620790496 (+6 ms)Region opened successfully at 1733620790500 (+4 ms) 2024-12-08T01:19:50,500 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d41be4f95858ff5b8c90ef34d6d4282, disabling compactions & flushes 2024-12-08T01:19:50,500 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,500 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,500 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. after waiting 0 ms 2024-12-08T01:19:50,500 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,500 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,500 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d41be4f95858ff5b8c90ef34d6d4282: Waiting for close lock at 1733620790500Disabling compacts and flushes for region at 1733620790500Disabling writes for close at 1733620790500Writing region close event to WAL at 1733620790500Closed at 1733620790500 2024-12-08T01:19:50,503 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-24241128/hregion-24241128.1733620790429 not finished, retry = 0 2024-12-08T01:19:50,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741930_1109 (size=175) 2024-12-08T01:19:50,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741930_1109 (size=175) 2024-12-08T01:19:50,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741930_1109 (size=175) 2024-12-08T01:19:50,606 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:50,606 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24241128:(num 1733620790429) 2024-12-08T01:19:50,606 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:50,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:50,623 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609, exclude list is [], retry=0 2024-12-08T01:19:50,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:50,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:50,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:50,646 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 2024-12-08T01:19:50,647 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38357:38357)] 2024-12-08T01:19:50,647 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d41be4f95858ff5b8c90ef34d6d4282, NAME => 'testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:50,647 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:50,647 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,647 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,648 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,649 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName a 2024-12-08T01:19:50,649 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,649 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,649 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,650 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName b 2024-12-08T01:19:50,650 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,650 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,650 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,651 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName c 2024-12-08T01:19:50,651 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,651 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,653 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,654 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,654 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,654 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:50,655 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,656 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d41be4f95858ff5b8c90ef34d6d4282; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59248954, jitterRate=-0.11712178587913513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:50,656 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d41be4f95858ff5b8c90ef34d6d4282: Writing region info on filesystem at 1733620790647Initializing all the Stores at 1733620790648 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790648Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790648Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790648Cleaning up temporary data from old regions at 1733620790654 (+6 ms)Region opened successfully at 1733620790656 (+2 ms) 2024-12-08T01:19:50,663 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d41be4f95858ff5b8c90ef34d6d4282 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-08T01:19:50,676 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/a/ba187560214b485288be880bc2cde8e9 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733620790656/Put/seqid=0 2024-12-08T01:19:50,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741933_1112 (size=5958) 2024-12-08T01:19:50,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741933_1112 (size=5958) 2024-12-08T01:19:50,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741933_1112 (size=5958) 2024-12-08T01:19:50,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/a/ba187560214b485288be880bc2cde8e9 2024-12-08T01:19:50,687 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/a/ba187560214b485288be880bc2cde8e9 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/a/ba187560214b485288be880bc2cde8e9 2024-12-08T01:19:50,691 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/a/ba187560214b485288be880bc2cde8e9, entries=10, sequenceid=13, filesize=5.8 K 2024-12-08T01:19:50,692 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9d41be4f95858ff5b8c90ef34d6d4282 in 29ms, sequenceid=13, compaction requested=false 2024-12-08T01:19:50,692 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9d41be4f95858ff5b8c90ef34d6d4282: 2024-12-08T01:19:50,707 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d41be4f95858ff5b8c90ef34d6d4282, disabling compactions & flushes 2024-12-08T01:19:50,707 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,707 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,707 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. after waiting 0 ms 2024-12-08T01:19:50,707 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,707 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,707 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:50,708 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d41be4f95858ff5b8c90ef34d6d4282: Waiting for close lock at 1733620790707Disabling compacts and flushes for region at 1733620790707Disabling writes for close at 1733620790707Writing region close event to WAL at 1733620790707Closed at 1733620790707 2024-12-08T01:19:50,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741932_1111 (size=6243) 2024-12-08T01:19:50,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741932_1111 (size=6243) 2024-12-08T01:19:50,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741932_1111 (size=6243) 2024-12-08T01:19:50,723 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609, size=6.1 K (6243bytes) 2024-12-08T01:19:50,723 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 2024-12-08T01:19:50,723 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 after 0ms 2024-12-08T01:19:50,745 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:50,745 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 took 22ms 2024-12-08T01:19:50,747 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 so closing down 2024-12-08T01:19:50,747 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:50,748 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733620790609.temp 2024-12-08T01:19:50,770 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000003-wal.1733620790609.temp 2024-12-08T01:19:50,771 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:50,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741934_1113 (size=5575) 2024-12-08T01:19:50,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741934_1113 (size=5575) 2024-12-08T01:19:50,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741934_1113 (size=5575) 2024-12-08T01:19:50,779 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000003-wal.1733620790609.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:50,780 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000003-wal.1733620790609.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000035 2024-12-08T01:19:50,781 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-08T01:19:50,781 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609, journal: Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609, size=6.1 K (6243bytes) at 1733620790723Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 so closing down at 1733620790747 (+24 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000003-wal.1733620790609.temp at 1733620790770 (+23 ms)3 split writer threads finished at 1733620790771 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000003-wal.1733620790609.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733620790779 (+8 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000003-wal.1733620790609.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000035 at 1733620790780 (+1 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733620790781 (+1 ms) 2024-12-08T01:19:50,782 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790609 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620790609 2024-12-08T01:19:50,783 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000035 2024-12-08T01:19:50,783 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:50,785 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:50,796 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785, exclude list is [], retry=0 2024-12-08T01:19:50,798 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:50,798 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:50,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:50,821 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 2024-12-08T01:19:50,821 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:33735:33735)] 2024-12-08T01:19:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d41be4f95858ff5b8c90ef34d6d4282, NAME => 'testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.', STARTKEY => '', ENDKEY => ''} 2024-12-08T01:19:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,823 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,823 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName a 2024-12-08T01:19:50,823 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,830 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/a/ba187560214b485288be880bc2cde8e9 2024-12-08T01:19:50,830 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,830 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,831 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName b 2024-12-08T01:19:50,831 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,832 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,832 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,832 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName c 2024-12-08T01:19:50,833 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:50,833 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:50,833 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,834 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,835 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000035 2024-12-08T01:19:50,858 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:50,859 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000035 2024-12-08T01:19:50,859 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d41be4f95858ff5b8c90ef34d6d4282 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-08T01:19:50,872 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/b/cb2e1f57eac54b479a0ad9f8965b180b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733620790692/Put/seqid=0 2024-12-08T01:19:50,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741936_1115 (size=5958) 2024-12-08T01:19:50,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741936_1115 (size=5958) 2024-12-08T01:19:50,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741936_1115 (size=5958) 2024-12-08T01:19:50,879 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/b/cb2e1f57eac54b479a0ad9f8965b180b 2024-12-08T01:19:50,896 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/c/bfcdab81399b4a37877024ad8e63f7f6 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733620790698/Put/seqid=0 2024-12-08T01:19:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741937_1116 (size=5958) 2024-12-08T01:19:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741937_1116 (size=5958) 2024-12-08T01:19:50,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741937_1116 (size=5958) 2024-12-08T01:19:50,903 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/c/bfcdab81399b4a37877024ad8e63f7f6 2024-12-08T01:19:50,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/b/cb2e1f57eac54b479a0ad9f8965b180b as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/b/cb2e1f57eac54b479a0ad9f8965b180b 2024-12-08T01:19:50,914 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/b/cb2e1f57eac54b479a0ad9f8965b180b, entries=10, sequenceid=35, filesize=5.8 K 2024-12-08T01:19:50,915 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/c/bfcdab81399b4a37877024ad8e63f7f6 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/c/bfcdab81399b4a37877024ad8e63f7f6 2024-12-08T01:19:50,920 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/c/bfcdab81399b4a37877024ad8e63f7f6, entries=10, sequenceid=35, filesize=5.8 K 2024-12-08T01:19:50,920 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 9d41be4f95858ff5b8c90ef34d6d4282 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-12-08T01:19:50,921 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000035 2024-12-08T01:19:50,922 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,922 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,923 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:50,924 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:50,926 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-08T01:19:50,927 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d41be4f95858ff5b8c90ef34d6d4282; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64216490, jitterRate=-0.04309973120689392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:50,927 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d41be4f95858ff5b8c90ef34d6d4282: Writing region info on filesystem at 1733620790822Initializing all the Stores at 1733620790822Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790822Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790823 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620790823Obtaining lock to block concurrent updates at 1733620790859 (+36 ms)Preparing flush snapshotting stores in 9d41be4f95858ff5b8c90ef34d6d4282 at 1733620790859Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733620790859Flushing stores of testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. at 1733620790859Flushing 9d41be4f95858ff5b8c90ef34d6d4282/b: creating writer at 1733620790859Flushing 9d41be4f95858ff5b8c90ef34d6d4282/b: appending metadata at 1733620790872 (+13 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/b: closing flushed file at 1733620790872Flushing 9d41be4f95858ff5b8c90ef34d6d4282/c: creating writer at 1733620790883 (+11 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/c: appending metadata at 1733620790895 (+12 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/c: closing flushed file at 1733620790895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb60a7b: reopening flushed file at 1733620790908 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cd454f7: reopening flushed file at 1733620790914 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 9d41be4f95858ff5b8c90ef34d6d4282 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1733620790920 (+6 ms)Cleaning up temporary data from old regions at 1733620790922 (+2 ms)Region opened successfully at 1733620790927 (+5 ms) 2024-12-08T01:19:51,005 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785, size=0 (0bytes) 2024-12-08T01:19:51,005 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 might be still open, length is 0 2024-12-08T01:19:51,005 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 2024-12-08T01:19:51,006 WARN [IPC Server handler 3 on default port 35931 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-08T01:19:51,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 after 1ms 2024-12-08T01:19:52,824 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:38222 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38222 dst: /127.0.0.1:41533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41533 remote=/127.0.0.1:38222]. Total timeout mills is 60000, 58133 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:52,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:41822 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41822 dst: /127.0.0.1:39867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:52,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1884899329_22 at /127.0.0.1:47618 [Receiving block BP-1381908131-172.17.0.2-1733620750211:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39611:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47618 dst: /127.0.0.1:39611 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:52,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741935_1117 (size=5567) 2024-12-08T01:19:52,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741935_1117 (size=5567) 2024-12-08T01:19:52,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741935_1117 (size=5567) 2024-12-08T01:19:54,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-08T01:19:54,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:54,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-08T01:19:55,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:55,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-08T01:19:55,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-08T01:19:55,007 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 after 4002ms 2024-12-08T01:19:55,034 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:55,034 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 took 4029ms 2024-12-08T01:19:55,036 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785; continuing. 2024-12-08T01:19:55,037 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 so closing down 2024-12-08T01:19:55,037 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-08T01:19:55,038 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733620790785.temp 2024-12-08T01:19:55,062 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000037-wal.1733620790785.temp 2024-12-08T01:19:55,063 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-08T01:19:55,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741938_1118 (size=5575) 2024-12-08T01:19:55,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741938_1118 (size=5575) 2024-12-08T01:19:55,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741938_1118 (size=5575) 2024-12-08T01:19:55,069 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000037-wal.1733620790785.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-08T01:19:55,070 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000037-wal.1733620790785.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000066 2024-12-08T01:19:55,070 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785, size=0, length=0, corrupted=false, cancelled=false 2024-12-08T01:19:55,070 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785, journal: Splitting hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785, size=0 (0bytes) at 1733620791005Finishing writing output for hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 so closing down at 1733620795037 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000037-wal.1733620790785.temp at 1733620795062 (+25 ms)3 split writer threads finished at 1733620795063 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000037-wal.1733620790785.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733620795069 (+6 ms)Rename recovered edits hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000037-wal.1733620790785.temp to hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000066 at 1733620795070 (+1 ms)Processed 30 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785, size=0, length=0, corrupted=false, cancelled=false at 1733620795070 2024-12-08T01:19:55,072 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 to hdfs://localhost:35931/hbase/oldWALs/wal.1733620790785 2024-12-08T01:19:55,072 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000066 2024-12-08T01:19:55,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-08T01:19:55,074 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35931/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423, archiveDir=hdfs://localhost:35931/hbase/oldWALs, maxLogs=32 2024-12-08T01:19:55,085 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620795074, exclude list is [], retry=0 2024-12-08T01:19:55,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41533,DS-39851136-a6e8-487a-a185-d8070836f45c,DISK] 2024-12-08T01:19:55,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39611,DS-a0fd70eb-79a9-4215-9ec6-781bc02562e3,DISK] 2024-12-08T01:19:55,088 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39867,DS-c4fd417a-751e-4030-9465-13c368067924,DISK] 2024-12-08T01:19:55,112 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620795074 2024-12-08T01:19:55,112 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38357:38357),(127.0.0.1/127.0.0.1:33735:33735),(127.0.0.1/127.0.0.1:39561:39561)] 2024-12-08T01:19:55,113 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T01:19:55,114 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,115 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName a 2024-12-08T01:19:55,115 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:55,120 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/a/ba187560214b485288be880bc2cde8e9 2024-12-08T01:19:55,120 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:55,121 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,121 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName b 2024-12-08T01:19:55,121 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:55,126 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/b/cb2e1f57eac54b479a0ad9f8965b180b 2024-12-08T01:19:55,126 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:55,126 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,127 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d41be4f95858ff5b8c90ef34d6d4282 columnFamilyName c 2024-12-08T01:19:55,127 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T01:19:55,132 DEBUG [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/c/bfcdab81399b4a37877024ad8e63f7f6 2024-12-08T01:19:55,132 INFO [StoreOpener-9d41be4f95858ff5b8c90ef34d6d4282-1 {}] regionserver.HStore(327): Store=9d41be4f95858ff5b8c90ef34d6d4282/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T01:19:55,132 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,133 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,134 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,135 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000066 2024-12-08T01:19:55,157 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-08T01:19:55,160 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000066 2024-12-08T01:19:55,160 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d41be4f95858ff5b8c90ef34d6d4282 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-08T01:19:55,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/a/8ff4341027cc419697d60e590dc36815 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733620790933/Put/seqid=0 2024-12-08T01:19:55,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741940_1120 (size=5958) 2024-12-08T01:19:55,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741940_1120 (size=5958) 2024-12-08T01:19:55,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741940_1120 (size=5958) 2024-12-08T01:19:55,180 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/a/8ff4341027cc419697d60e590dc36815 2024-12-08T01:19:55,196 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/b/f129e08faf234aa8973c77012cc324ad is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733620790939/Put/seqid=0 2024-12-08T01:19:55,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741941_1121 (size=5958) 2024-12-08T01:19:55,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741941_1121 (size=5958) 2024-12-08T01:19:55,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741941_1121 (size=5958) 2024-12-08T01:19:55,202 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/b/f129e08faf234aa8973c77012cc324ad 2024-12-08T01:19:55,219 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/c/f4c087f2f5f148ebaf5d14619481ce02 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733620790947/Put/seqid=0 2024-12-08T01:19:55,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741942_1122 (size=5958) 2024-12-08T01:19:55,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741942_1122 (size=5958) 2024-12-08T01:19:55,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741942_1122 (size=5958) 2024-12-08T01:19:55,225 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/c/f4c087f2f5f148ebaf5d14619481ce02 2024-12-08T01:19:55,229 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/a/8ff4341027cc419697d60e590dc36815 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/a/8ff4341027cc419697d60e590dc36815 2024-12-08T01:19:55,234 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/a/8ff4341027cc419697d60e590dc36815, entries=10, sequenceid=66, filesize=5.8 K 2024-12-08T01:19:55,235 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/b/f129e08faf234aa8973c77012cc324ad as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/b/f129e08faf234aa8973c77012cc324ad 2024-12-08T01:19:55,240 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/b/f129e08faf234aa8973c77012cc324ad, entries=10, sequenceid=66, filesize=5.8 K 2024-12-08T01:19:55,241 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/.tmp/c/f4c087f2f5f148ebaf5d14619481ce02 as hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/c/f4c087f2f5f148ebaf5d14619481ce02 2024-12-08T01:19:55,246 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/c/f4c087f2f5f148ebaf5d14619481ce02, entries=10, sequenceid=66, filesize=5.8 K 2024-12-08T01:19:55,246 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9d41be4f95858ff5b8c90ef34d6d4282 in 86ms, sequenceid=66, compaction requested=false; wal=null 2024-12-08T01:19:55,247 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/0000000000000000066 2024-12-08T01:19:55,248 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,248 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,249 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-08T01:19:55,250 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9d41be4f95858ff5b8c90ef34d6d4282 2024-12-08T01:19:55,252 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/hbase/data/default/testReplayEditsWrittenViaHRegion/9d41be4f95858ff5b8c90ef34d6d4282/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-08T01:19:55,253 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9d41be4f95858ff5b8c90ef34d6d4282; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59943493, jitterRate=-0.10677234828472137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-08T01:19:55,254 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9d41be4f95858ff5b8c90ef34d6d4282: Writing region info on filesystem at 1733620795113Initializing all the Stores at 1733620795114 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620795114Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620795114Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733620795114Obtaining lock to block concurrent updates at 1733620795160 (+46 ms)Preparing flush snapshotting stores in 9d41be4f95858ff5b8c90ef34d6d4282 at 1733620795160Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733620795161 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. at 1733620795161Flushing 9d41be4f95858ff5b8c90ef34d6d4282/a: creating writer at 1733620795161Flushing 9d41be4f95858ff5b8c90ef34d6d4282/a: appending metadata at 1733620795173 (+12 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/a: closing flushed file at 1733620795173Flushing 9d41be4f95858ff5b8c90ef34d6d4282/b: creating writer at 1733620795184 (+11 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/b: appending metadata at 1733620795196 (+12 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/b: closing flushed file at 1733620795196Flushing 9d41be4f95858ff5b8c90ef34d6d4282/c: creating writer at 1733620795206 (+10 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/c: appending metadata at 1733620795218 (+12 ms)Flushing 9d41be4f95858ff5b8c90ef34d6d4282/c: closing flushed file at 1733620795218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a1aac7a: reopening flushed file at 1733620795229 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54e8f722: reopening flushed file at 1733620795234 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5577c4ed: reopening flushed file at 1733620795240 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9d41be4f95858ff5b8c90ef34d6d4282 in 86ms, sequenceid=66, compaction requested=false; wal=null at 1733620795246 (+6 ms)Cleaning up temporary data from old regions at 1733620795248 (+2 ms)Region opened successfully at 1733620795253 (+5 ms) 2024-12-08T01:19:55,266 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9d41be4f95858ff5b8c90ef34d6d4282, disabling compactions & flushes 2024-12-08T01:19:55,267 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:55,267 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:55,267 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. after waiting 0 ms 2024-12-08T01:19:55,267 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:55,268 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733620790424.9d41be4f95858ff5b8c90ef34d6d4282. 2024-12-08T01:19:55,268 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9d41be4f95858ff5b8c90ef34d6d4282: Waiting for close lock at 1733620795266Disabling compacts and flushes for region at 1733620795266Disabling writes for close at 1733620795267 (+1 ms)Writing region close event to WAL at 1733620795268 (+1 ms)Closed at 1733620795268 2024-12-08T01:19:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741939_1119 (size=175) 2024-12-08T01:19:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741939_1119 (size=175) 2024-12-08T01:19:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741939_1119 (size=175) 2024-12-08T01:19:55,272 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-08T01:19:55,272 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733620795074) 2024-12-08T01:19:55,283 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=450 (was 441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674358634_22 at /127.0.0.1:47662 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35931 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674358634_22 at /127.0.0.1:38264 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35931 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674358634_22 at /127.0.0.1:41856 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1345) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=165 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=9752 (was 9788) 2024-12-08T01:19:55,284 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-08T01:19:55,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T01:19:55,284 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:19:55,284 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:55,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T01:19:55,285 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T01:19:55,285 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1285900214, stopped=false 2024-12-08T01:19:55,285 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=96982154c400,39725,1733620754685 2024-12-08T01:19:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T01:19:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:55,334 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T01:19:55,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T01:19:55,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:55,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:55,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:55,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T01:19:55,337 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,37871,1733620755662' ***** 2024-12-08T01:19:55,337 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T01:19:55,337 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '96982154c400,43235,1733620755742' ***** 2024-12-08T01:19:55,337 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T01:19:55,338 INFO [RS:2;96982154c400:43235 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:19:55,338 INFO [RS:1;96982154c400:37871 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T01:19:55,338 INFO [RS:2;96982154c400:43235 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T01:19:55,338 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:19:55,338 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T01:19:55,339 INFO [RS:2;96982154c400:43235 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T01:19:55,339 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(959): stopping server 96982154c400,43235,1733620755742 2024-12-08T01:19:55,339 INFO [RS:1;96982154c400:37871 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T01:19:55,339 INFO [RS:1;96982154c400:37871 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T01:19:55,339 INFO [RS:2;96982154c400:43235 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:55,339 INFO [RS:2;96982154c400:43235 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;96982154c400:43235. 2024-12-08T01:19:55,339 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:55,339 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(959): stopping server 96982154c400,37871,1733620755662 2024-12-08T01:19:55,339 DEBUG [RS:2;96982154c400:43235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:55,339 INFO [RS:1;96982154c400:37871 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:55,339 DEBUG [RS:2;96982154c400:43235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,339 INFO [RS:1;96982154c400:37871 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;96982154c400:37871. 2024-12-08T01:19:55,339 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b9fc8dd8c927db706d5818cc32f0abc, disabling compactions & flushes 2024-12-08T01:19:55,339 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(976): stopping server 96982154c400,43235,1733620755742; all regions closed. 2024-12-08T01:19:55,339 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:55,339 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:55,340 DEBUG [RS:1;96982154c400:37871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T01:19:55,340 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. after waiting 0 ms 2024-12-08T01:19:55,340 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:55,340 DEBUG [RS:1;96982154c400:37871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,340 INFO [RS:1;96982154c400:37871 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:19:55,340 INFO [RS:1;96982154c400:37871 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:19:55,340 INFO [RS:1;96982154c400:37871 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:19:55,340 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T01:19:55,341 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-08T01:19:55,341 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1325): Online Regions={7b9fc8dd8c927db706d5818cc32f0abc=testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc., 1588230740=hbase:meta,,1.1588230740} 2024-12-08T01:19:55,341 DEBUG [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7b9fc8dd8c927db706d5818cc32f0abc 2024-12-08T01:19:55,341 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T01:19:55,341 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T01:19:55,341 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T01:19:55,341 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T01:19:55,341 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T01:19:55,341 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-08T01:19:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741833_1009 (size=175) 2024-12-08T01:19:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741833_1009 (size=175) 2024-12-08T01:19:55,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741833_1009 (size=175) 2024-12-08T01:19:55,345 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/default/testReplayEditsAfterRegionMovedWithMultiCF/7b9fc8dd8c927db706d5818cc32f0abc/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-08T01:19:55,346 DEBUG [RS:2;96982154c400:43235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs 2024-12-08T01:19:55,346 INFO [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C43235%2C1733620755742:(num 1733620757260) 2024-12-08T01:19:55,346 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b9fc8dd8c927db706d5818cc32f0abc: Waiting for close lock at 1733620795339Running coprocessor pre-close hooks at 1733620795339Disabling compacts and flushes for region at 1733620795339Disabling writes for close at 1733620795340 (+1 ms)Writing region close event to WAL at 1733620795341 (+1 ms)Running coprocessor post-close hooks at 1733620795346 (+5 ms)Closed at 1733620795346 2024-12-08T01:19:55,346 DEBUG [RS:2;96982154c400:43235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:55,346 DEBUG [RS_CLOSE_REGION-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc. 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:55,346 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:55,346 INFO [regionserver/96982154c400:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T01:19:55,346 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T01:19:55,346 INFO [RS:2;96982154c400:43235 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:55,347 INFO [RS:2;96982154c400:43235 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43235 2024-12-08T01:19:55,347 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/info/c9497e7d4d4a41fd89995f4ad64f07df is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733620770585.7b9fc8dd8c927db706d5818cc32f0abc./info:regioninfo/1733620774595/Put/seqid=0 2024-12-08T01:19:55,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,43235,1733620755742 2024-12-08T01:19:55,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741943_1123 (size=6778) 2024-12-08T01:19:55,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:19:55,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741943_1123 (size=6778) 2024-12-08T01:19:55,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741943_1123 (size=6778) 2024-12-08T01:19:55,355 INFO [RS:2;96982154c400:43235 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:55,356 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,43235,1733620755742] 2024-12-08T01:19:55,356 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/info/c9497e7d4d4a41fd89995f4ad64f07df 2024-12-08T01:19:55,360 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/.tmp/info/c9497e7d4d4a41fd89995f4ad64f07df as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/info/c9497e7d4d4a41fd89995f4ad64f07df 2024-12-08T01:19:55,364 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/info/c9497e7d4d4a41fd89995f4ad64f07df, entries=8, sequenceid=23, filesize=6.6 K 2024-12-08T01:19:55,365 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-12-08T01:19:55,365 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T01:19:55,370 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-08T01:19:55,371 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T01:19:55,371 INFO [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:55,371 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733620795341Running coprocessor pre-close hooks at 1733620795341Disabling compacts and flushes for region at 1733620795341Disabling writes for close at 1733620795341Obtaining lock to block concurrent updates at 1733620795341Preparing flush snapshotting stores in 1588230740 at 1733620795341Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733620795342 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733620795343 (+1 ms)Flushing 1588230740/info: creating writer at 1733620795343Flushing 1588230740/info: appending metadata at 1733620795347 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733620795347Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cd1f4e5: reopening flushed file at 1733620795360 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1733620795365 (+5 ms)Writing region close event to WAL at 1733620795366 (+1 ms)Running coprocessor post-close hooks at 1733620795371 (+5 ms)Closed at 1733620795371 2024-12-08T01:19:55,371 DEBUG [RS_CLOSE_META-regionserver/96982154c400:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T01:19:55,375 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,43235,1733620755742 already deleted, retry=false 2024-12-08T01:19:55,376 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 96982154c400,43235,1733620755742 expired; onlineServers=1 2024-12-08T01:19:55,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:55,466 INFO [RS:2;96982154c400:43235 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:55,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43235-0x10003016cce0003, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:55,466 INFO [RS:2;96982154c400:43235 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,43235,1733620755742; zookeeper connection closed. 2024-12-08T01:19:55,466 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57f03c4e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57f03c4e 2024-12-08T01:19:55,541 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(976): stopping server 96982154c400,37871,1733620755662; all regions closed. 2024-12-08T01:19:55,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741893_1071 (size=2625) 2024-12-08T01:19:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741893_1071 (size=2625) 2024-12-08T01:19:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741893_1071 (size=2625) 2024-12-08T01:19:55,549 DEBUG [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs 2024-12-08T01:19:55,549 INFO [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C37871%2C1733620755662.meta:.meta(num 1733620773712) 2024-12-08T01:19:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741835_1011 (size=3621) 2024-12-08T01:19:55,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741835_1011 (size=3621) 2024-12-08T01:19:55,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741835_1011 (size=3621) 2024-12-08T01:19:55,555 DEBUG [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/oldWALs 2024-12-08T01:19:55,555 INFO [RS:1;96982154c400:37871 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 96982154c400%2C37871%2C1733620755662:(num 1733620757260) 2024-12-08T01:19:55,555 DEBUG [RS:1;96982154c400:37871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T01:19:55,555 INFO [RS:1;96982154c400:37871 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T01:19:55,555 INFO [RS:1;96982154c400:37871 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:55,555 INFO [RS:1;96982154c400:37871 {}] hbase.ChoreService(370): Chore service for: regionserver/96982154c400:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:55,555 INFO [RS:1;96982154c400:37871 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:55,555 INFO [regionserver/96982154c400:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:55,556 INFO [RS:1;96982154c400:37871 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37871 2024-12-08T01:19:55,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T01:19:55,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/96982154c400,37871,1733620755662 2024-12-08T01:19:55,565 INFO [RS:1;96982154c400:37871 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:55,566 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [96982154c400,37871,1733620755662] 2024-12-08T01:19:55,586 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/96982154c400,37871,1733620755662 already deleted, retry=false 2024-12-08T01:19:55,586 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 96982154c400,37871,1733620755662 expired; onlineServers=0 2024-12-08T01:19:55,587 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '96982154c400,39725,1733620754685' ***** 2024-12-08T01:19:55,587 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T01:19:55,587 INFO [M:0;96982154c400:39725 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T01:19:55,587 INFO [M:0;96982154c400:39725 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T01:19:55,588 DEBUG [M:0;96982154c400:39725 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T01:19:55,588 DEBUG [M:0;96982154c400:39725 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T01:19:55,588 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T01:19:55,588 DEBUG [master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620756902 {}] cleaner.HFileCleaner(306): Exit Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.large.0-1733620756902,5,FailOnTimeoutGroup] 2024-12-08T01:19:55,588 DEBUG [master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620756904 {}] cleaner.HFileCleaner(306): Exit Thread[master/96982154c400:0:becomeActiveMaster-HFileCleaner.small.0-1733620756904,5,FailOnTimeoutGroup] 2024-12-08T01:19:55,589 INFO [M:0;96982154c400:39725 {}] hbase.ChoreService(370): Chore service for: master/96982154c400:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T01:19:55,589 INFO [M:0;96982154c400:39725 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T01:19:55,589 DEBUG [M:0;96982154c400:39725 {}] master.HMaster(1795): Stopping service threads 2024-12-08T01:19:55,589 INFO [M:0;96982154c400:39725 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T01:19:55,589 INFO [M:0;96982154c400:39725 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T01:19:55,591 INFO [M:0;96982154c400:39725 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T01:19:55,591 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T01:19:55,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T01:19:55,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T01:19:55,597 DEBUG [M:0;96982154c400:39725 {}] zookeeper.ZKUtil(347): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T01:19:55,597 WARN [M:0;96982154c400:39725 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T01:19:55,598 INFO [M:0;96982154c400:39725 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/.lastflushedseqids 2024-12-08T01:19:55,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741944_1124 (size=119) 2024-12-08T01:19:55,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741944_1124 (size=119) 2024-12-08T01:19:55,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741944_1124 (size=119) 2024-12-08T01:19:55,615 INFO [M:0;96982154c400:39725 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T01:19:55,615 INFO [M:0;96982154c400:39725 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T01:19:55,616 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T01:19:55,616 INFO [M:0;96982154c400:39725 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:55,616 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:55,616 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T01:19:55,616 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:55,616 INFO [M:0;96982154c400:39725 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-12-08T01:19:55,629 DEBUG [M:0;96982154c400:39725 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4783a06328445b49256043bf00dea0b is 82, key is hbase:meta,,1/info:regioninfo/1733620773904/Put/seqid=0 2024-12-08T01:19:55,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741945_1125 (size=6063) 2024-12-08T01:19:55,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741945_1125 (size=6063) 2024-12-08T01:19:55,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741945_1125 (size=6063) 2024-12-08T01:19:55,634 INFO [M:0;96982154c400:39725 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4783a06328445b49256043bf00dea0b 2024-12-08T01:19:55,650 DEBUG [M:0;96982154c400:39725 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/101c495687004b60b148d2ff9132fe78 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733620771009/Put/seqid=0 2024-12-08T01:19:55,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741946_1126 (size=7907) 2024-12-08T01:19:55,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741946_1126 (size=7907) 2024-12-08T01:19:55,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741946_1126 (size=7907) 2024-12-08T01:19:55,656 INFO [M:0;96982154c400:39725 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/101c495687004b60b148d2ff9132fe78 2024-12-08T01:19:55,660 INFO [M:0;96982154c400:39725 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 101c495687004b60b148d2ff9132fe78 2024-12-08T01:19:55,672 DEBUG [M:0;96982154c400:39725 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d363843efe60465caa8e835fbfae1db8 is 69, key is 96982154c400,37871,1733620755662/rs:state/1733620756967/Put/seqid=0 2024-12-08T01:19:55,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:55,676 INFO [RS:1;96982154c400:37871 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:55,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37871-0x10003016cce0002, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:55,676 INFO [RS:1;96982154c400:37871 {}] regionserver.HRegionServer(1031): Exiting; stopping=96982154c400,37871,1733620755662; zookeeper connection closed. 2024-12-08T01:19:55,676 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ef437ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ef437ae 2024-12-08T01:19:55,676 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-08T01:19:55,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741947_1127 (size=5440) 2024-12-08T01:19:55,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741947_1127 (size=5440) 2024-12-08T01:19:55,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741947_1127 (size=5440) 2024-12-08T01:19:55,678 INFO [M:0;96982154c400:39725 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d363843efe60465caa8e835fbfae1db8 2024-12-08T01:19:55,682 INFO [M:0;96982154c400:39725 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d363843efe60465caa8e835fbfae1db8 2024-12-08T01:19:55,683 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4783a06328445b49256043bf00dea0b as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4783a06328445b49256043bf00dea0b 2024-12-08T01:19:55,687 INFO [M:0;96982154c400:39725 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4783a06328445b49256043bf00dea0b, entries=14, sequenceid=207, filesize=5.9 K 2024-12-08T01:19:55,688 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/101c495687004b60b148d2ff9132fe78 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/101c495687004b60b148d2ff9132fe78 2024-12-08T01:19:55,693 INFO [M:0;96982154c400:39725 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 101c495687004b60b148d2ff9132fe78 2024-12-08T01:19:55,693 INFO [M:0;96982154c400:39725 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/101c495687004b60b148d2ff9132fe78, entries=21, sequenceid=207, filesize=7.7 K 2024-12-08T01:19:55,694 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d363843efe60465caa8e835fbfae1db8 as hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d363843efe60465caa8e835fbfae1db8 2024-12-08T01:19:55,698 INFO [M:0;96982154c400:39725 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d363843efe60465caa8e835fbfae1db8 2024-12-08T01:19:55,699 INFO [M:0;96982154c400:39725 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35931/user/jenkins/test-data/6f9c6039-6cdf-13a1-a7ab-c752d0365eae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d363843efe60465caa8e835fbfae1db8, entries=3, sequenceid=207, filesize=5.3 K 2024-12-08T01:19:55,700 INFO [M:0;96982154c400:39725 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=207, compaction requested=false 2024-12-08T01:19:55,701 INFO [M:0;96982154c400:39725 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T01:19:55,701 DEBUG [M:0;96982154c400:39725 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733620795616Disabling compacts and flushes for region at 1733620795616Disabling writes for close at 1733620795616Obtaining lock to block concurrent updates at 1733620795616Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733620795616Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1733620795616Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733620795617 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733620795617Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733620795628 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733620795628Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733620795638 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733620795650 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733620795650Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733620795660 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733620795671 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733620795671Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2ca49b: reopening flushed file at 1733620795682 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1236c5a: reopening flushed file at 1733620795687 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@116449ea: reopening flushed file at 1733620795693 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=207, compaction requested=false at 1733620795700 (+7 ms)Writing region close event to WAL at 1733620795701 (+1 ms)Closed at 1733620795701 2024-12-08T01:19:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39867 is added to blk_1073741830_1006 (size=102162) 2024-12-08T01:19:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41533 is added to blk_1073741830_1006 (size=102162) 2024-12-08T01:19:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39611 is added to blk_1073741830_1006 (size=102162) 2024-12-08T01:19:55,703 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T01:19:55,703 INFO [M:0;96982154c400:39725 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T01:19:55,704 INFO [M:0;96982154c400:39725 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39725 2024-12-08T01:19:55,704 INFO [M:0;96982154c400:39725 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T01:19:55,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:55,813 INFO [M:0;96982154c400:39725 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T01:19:55,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39725-0x10003016cce0000, quorum=127.0.0.1:54213, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T01:19:55,823 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620784876 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620784876 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733620775595/wal.1733620784876 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:55,826 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620775013 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620775013 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733620774655/wal.1733620775013 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:55,827 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733620766095/wal.1733620766212 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:55,829 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785715 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785715 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733620785111/wal.1733620785715 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:55,829 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733620785893/wal.1733620785967 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:55,829 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733620790423/wal.1733620790785 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:55,830 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733620759405/wal.1733620759490 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T01:19:55,831 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal.1733620759227 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal.1733620759227 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733620758783/wal.1733620759227 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:55,834 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal.1733620758525 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal.1733620758525 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733620758117/wal.1733620758525 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1884899329_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-08T01:19:55,838 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:55,840 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:55,841 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:55,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:55,841 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:55,844 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:19:55,844 WARN [BP-1381908131-172.17.0.2-1733620750211 heartbeating to localhost/127.0.0.1:35931 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:19:55,844 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:19:55,844 WARN [BP-1381908131-172.17.0.2-1733620750211 heartbeating to localhost/127.0.0.1:35931 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1381908131-172.17.0.2-1733620750211 (Datanode Uuid 3b43399b-4a4d-46eb-aaeb-19c0ebf380e8) service to localhost/127.0.0.1:35931 2024-12-08T01:19:55,845 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data5/current/BP-1381908131-172.17.0.2-1733620750211 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:55,846 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data6/current/BP-1381908131-172.17.0.2-1733620750211 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:55,846 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:19:55,853 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:55,853 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:55,853 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:55,853 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:55,854 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:55,854 WARN [BP-1381908131-172.17.0.2-1733620750211 heartbeating to localhost/127.0.0.1:35931 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:19:55,854 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:19:55,855 WARN [BP-1381908131-172.17.0.2-1733620750211 heartbeating to localhost/127.0.0.1:35931 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1381908131-172.17.0.2-1733620750211 (Datanode Uuid 51479df6-55ff-431e-b610-6e5a5f4eaf1f) service to localhost/127.0.0.1:35931 2024-12-08T01:19:55,855 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:19:55,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data3/current/BP-1381908131-172.17.0.2-1733620750211 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:55,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data4/current/BP-1381908131-172.17.0.2-1733620750211 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:55,855 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:19:55,857 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T01:19:55,857 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:55,857 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:55,858 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:55,858 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:55,859 WARN [BP-1381908131-172.17.0.2-1733620750211 heartbeating to localhost/127.0.0.1:35931 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T01:19:55,859 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T01:19:55,859 WARN [BP-1381908131-172.17.0.2-1733620750211 heartbeating to localhost/127.0.0.1:35931 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1381908131-172.17.0.2-1733620750211 (Datanode Uuid 9d496315-77bf-4f0f-91fe-bfe62675722e) service to localhost/127.0.0.1:35931 2024-12-08T01:19:55,859 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T01:19:55,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data1/current/BP-1381908131-172.17.0.2-1733620750211 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:55,859 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/cluster_5fe234ea-5ff9-918d-c5ca-e6f4db00b0f9/data/data2/current/BP-1381908131-172.17.0.2-1733620750211 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T01:19:55,860 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T01:19:55,865 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T01:19:55,865 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T01:19:55,865 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T01:19:55,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T01:19:55,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fc938a70-0e65-da9c-6fec-8baa22d46f2e/hadoop.log.dir/,STOPPED} 2024-12-08T01:19:55,876 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T01:19:55,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down