2024-12-02 15:46:11,436 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-02 15:46:11,445 main DEBUG Took 0.007964 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 15:46:11,446 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 15:46:11,446 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 15:46:11,447 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 15:46:11,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,453 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 15:46:11,465 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,467 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,467 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,468 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,469 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,470 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,471 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,472 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,472 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,473 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 15:46:11,475 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,475 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 15:46:11,476 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 15:46:11,477 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 15:46:11,479 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 15:46:11,479 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 15:46:11,480 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 15:46:11,481 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 15:46:11,488 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 15:46:11,490 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 15:46:11,492 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 15:46:11,492 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 15:46:11,493 main DEBUG createAppenders(={Console}) 2024-12-02 15:46:11,493 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-02 15:46:11,493 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-02 15:46:11,494 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-02 15:46:11,495 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 15:46:11,495 main DEBUG OutputStream closed 2024-12-02 15:46:11,495 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 15:46:11,495 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 15:46:11,496 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-02 15:46:11,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 15:46:11,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 15:46:11,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 15:46:11,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 15:46:11,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 15:46:11,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 15:46:11,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 15:46:11,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 15:46:11,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 15:46:11,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 15:46:11,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 15:46:11,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 15:46:11,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 15:46:11,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 15:46:11,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 15:46:11,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 15:46:11,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 15:46:11,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 15:46:11,569 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 15:46:11,569 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-02 15:46:11,570 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 15:46:11,570 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-02T15:46:11,776 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c 2024-12-02 15:46:11,779 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 15:46:11,779 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T15:46:11,788 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-02T15:46:11,793 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-02T15:46:11,812 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T15:46:11,850 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T15:46:11,851 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T15:46:11,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T15:46:11,874 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658, deleteOnExit=true 2024-12-02T15:46:11,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T15:46:11,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/test.cache.data in system properties and HBase conf 2024-12-02T15:46:11,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T15:46:11,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir in system properties and HBase conf 2024-12-02T15:46:11,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T15:46:11,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T15:46:11,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T15:46:11,957 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T15:46:12,041 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T15:46:12,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T15:46:12,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T15:46:12,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T15:46:12,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T15:46:12,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T15:46:12,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T15:46:12,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T15:46:12,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T15:46:12,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T15:46:12,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/nfs.dump.dir in system properties and HBase conf 2024-12-02T15:46:12,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/java.io.tmpdir in system properties and HBase conf 2024-12-02T15:46:12,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T15:46:12,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T15:46:12,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T15:46:12,945 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T15:46:13,023 INFO [Time-limited test {}] log.Log(170): Logging initialized @2213ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T15:46:13,109 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T15:46:13,194 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T15:46:13,223 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T15:46:13,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T15:46:13,226 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T15:46:13,243 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T15:46:13,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,AVAILABLE} 2024-12-02T15:46:13,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T15:46:13,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/java.io.tmpdir/jetty-localhost-34597-hadoop-hdfs-3_4_1-tests_jar-_-any-8257331470575152055/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T15:46:13,483 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:34597} 2024-12-02T15:46:13,483 INFO [Time-limited test {}] server.Server(415): Started @2674ms 2024-12-02T15:46:13,970 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T15:46:13,976 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T15:46:13,977 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T15:46:13,977 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T15:46:13,977 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T15:46:13,978 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,AVAILABLE} 2024-12-02T15:46:13,979 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T15:46:14,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/java.io.tmpdir/jetty-localhost-41899-hadoop-hdfs-3_4_1-tests_jar-_-any-7128626205062219858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T15:46:14,073 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:41899} 2024-12-02T15:46:14,074 INFO [Time-limited test {}] server.Server(415): Started @3265ms 2024-12-02T15:46:14,122 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T15:46:14,229 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T15:46:14,242 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T15:46:14,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T15:46:14,245 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T15:46:14,245 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T15:46:14,247 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,AVAILABLE} 2024-12-02T15:46:14,248 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T15:46:14,377 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/java.io.tmpdir/jetty-localhost-44365-hadoop-hdfs-3_4_1-tests_jar-_-any-12383114103453163057/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T15:46:14,378 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:44365} 2024-12-02T15:46:14,378 INFO [Time-limited test {}] server.Server(415): Started @3569ms 2024-12-02T15:46:14,380 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T15:46:14,426 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T15:46:14,431 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T15:46:14,432 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T15:46:14,432 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T15:46:14,433 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T15:46:14,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,AVAILABLE} 2024-12-02T15:46:14,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T15:46:14,556 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/java.io.tmpdir/jetty-localhost-45221-hadoop-hdfs-3_4_1-tests_jar-_-any-17027954401879524078/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T15:46:14,557 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:45221} 2024-12-02T15:46:14,557 INFO [Time-limited test {}] server.Server(415): Started @3748ms 2024-12-02T15:46:14,559 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T15:46:15,314 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data1/current/BP-1466660644-172.17.0.3-1733154372547/current, will proceed with Du for space computation calculation, 2024-12-02T15:46:15,314 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data2/current/BP-1466660644-172.17.0.3-1733154372547/current, will proceed with Du for space computation calculation, 2024-12-02T15:46:15,330 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data3/current/BP-1466660644-172.17.0.3-1733154372547/current, will proceed with Du for space computation calculation, 2024-12-02T15:46:15,330 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data4/current/BP-1466660644-172.17.0.3-1733154372547/current, will proceed with Du for space computation calculation, 2024-12-02T15:46:15,353 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T15:46:15,353 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T15:46:15,398 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b691f31ef27fba8 with lease ID 0x48aaa65d82934aaa: Processing first storage report for DS-1266b24c-285f-4f76-a775-e330a6410b42 from datanode DatanodeRegistration(127.0.0.1:43105, datanodeUuid=c236a47e-8e13-4fb3-8647-23ed49cac4a3, infoPort=44449, infoSecurePort=0, ipcPort=41261, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547) 2024-12-02T15:46:15,399 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b691f31ef27fba8 with lease ID 0x48aaa65d82934aaa: from storage DS-1266b24c-285f-4f76-a775-e330a6410b42 node DatanodeRegistration(127.0.0.1:43105, datanodeUuid=c236a47e-8e13-4fb3-8647-23ed49cac4a3, infoPort=44449, infoSecurePort=0, ipcPort=41261, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T15:46:15,400 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe46d9ba6944d90d9 with lease ID 0x48aaa65d82934aab: Processing first storage report for DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f from datanode DatanodeRegistration(127.0.0.1:42367, datanodeUuid=51b79570-c33f-49df-95ef-06df9bdb5ba3, infoPort=33971, infoSecurePort=0, ipcPort=34277, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547) 2024-12-02T15:46:15,400 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe46d9ba6944d90d9 with lease ID 0x48aaa65d82934aab: from storage DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f node DatanodeRegistration(127.0.0.1:42367, datanodeUuid=51b79570-c33f-49df-95ef-06df9bdb5ba3, infoPort=33971, infoSecurePort=0, ipcPort=34277, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T15:46:15,400 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b691f31ef27fba8 with lease ID 0x48aaa65d82934aaa: Processing first storage report for DS-2ca01211-0625-4ba3-a4bc-52ed3b675a4a from datanode DatanodeRegistration(127.0.0.1:43105, datanodeUuid=c236a47e-8e13-4fb3-8647-23ed49cac4a3, infoPort=44449, infoSecurePort=0, ipcPort=41261, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547) 2024-12-02T15:46:15,400 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b691f31ef27fba8 with lease ID 0x48aaa65d82934aaa: from storage DS-2ca01211-0625-4ba3-a4bc-52ed3b675a4a node DatanodeRegistration(127.0.0.1:43105, datanodeUuid=c236a47e-8e13-4fb3-8647-23ed49cac4a3, infoPort=44449, infoSecurePort=0, ipcPort=41261, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T15:46:15,401 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe46d9ba6944d90d9 with lease ID 0x48aaa65d82934aab: Processing first storage report for DS-63ccb417-8924-41a9-8daa-5646e1e6148a from datanode DatanodeRegistration(127.0.0.1:42367, datanodeUuid=51b79570-c33f-49df-95ef-06df9bdb5ba3, infoPort=33971, infoSecurePort=0, ipcPort=34277, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547) 2024-12-02T15:46:15,401 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe46d9ba6944d90d9 with lease ID 0x48aaa65d82934aab: from storage DS-63ccb417-8924-41a9-8daa-5646e1e6148a node DatanodeRegistration(127.0.0.1:42367, datanodeUuid=51b79570-c33f-49df-95ef-06df9bdb5ba3, infoPort=33971, infoSecurePort=0, ipcPort=34277, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T15:46:15,404 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data5/current/BP-1466660644-172.17.0.3-1733154372547/current, will proceed with Du for space computation calculation, 2024-12-02T15:46:15,405 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data6/current/BP-1466660644-172.17.0.3-1733154372547/current, will proceed with Du for space computation calculation, 2024-12-02T15:46:15,425 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T15:46:15,431 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ab5fff918fe2954 with lease ID 0x48aaa65d82934aac: Processing first storage report for DS-6e826597-5c5b-4d40-98e7-7e5b93544faf from datanode DatanodeRegistration(127.0.0.1:38685, datanodeUuid=9cf4a451-2c25-42fd-8fe4-b2b5063f94ec, infoPort=34763, infoSecurePort=0, ipcPort=38529, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547) 2024-12-02T15:46:15,432 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ab5fff918fe2954 with lease ID 0x48aaa65d82934aac: from storage DS-6e826597-5c5b-4d40-98e7-7e5b93544faf node DatanodeRegistration(127.0.0.1:38685, datanodeUuid=9cf4a451-2c25-42fd-8fe4-b2b5063f94ec, infoPort=34763, infoSecurePort=0, ipcPort=38529, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T15:46:15,432 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2ab5fff918fe2954 with lease ID 0x48aaa65d82934aac: Processing first storage report for DS-239bf9aa-9015-4da7-a964-ec0cbff2aefc from datanode DatanodeRegistration(127.0.0.1:38685, datanodeUuid=9cf4a451-2c25-42fd-8fe4-b2b5063f94ec, infoPort=34763, infoSecurePort=0, ipcPort=38529, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547) 2024-12-02T15:46:15,433 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2ab5fff918fe2954 with lease ID 0x48aaa65d82934aac: from storage DS-239bf9aa-9015-4da7-a964-ec0cbff2aefc node DatanodeRegistration(127.0.0.1:38685, datanodeUuid=9cf4a451-2c25-42fd-8fe4-b2b5063f94ec, infoPort=34763, infoSecurePort=0, ipcPort=38529, storageInfo=lv=-57;cid=testClusterID;nsid=1685789903;c=1733154372547), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T15:46:15,452 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c 2024-12-02T15:46:15,512 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/zookeeper_0, clientPort=53141, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T15:46:15,521 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53141 2024-12-02T15:46:15,533 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:15,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:15,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741825_1001 (size=7) 2024-12-02T15:46:15,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741825_1001 (size=7) 2024-12-02T15:46:15,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741825_1001 (size=7) 2024-12-02T15:46:16,169 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82 with version=8 2024-12-02T15:46:16,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/hbase-staging 2024-12-02T15:46:16,409 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e1c0b6c2739c:0 server-side Connection retries=45 2024-12-02T15:46:16,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:16,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:16,423 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T15:46:16,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:16,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T15:46:16,556 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T15:46:16,610 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T15:46:16,622 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T15:46:16,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T15:46:16,658 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 71083 (auto-detected) 2024-12-02T15:46:16,659 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-02T15:46:16,684 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42629 2024-12-02T15:46:16,705 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42629 connecting to ZooKeeper ensemble=127.0.0.1:53141 2024-12-02T15:46:16,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426290x0, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T15:46:16,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42629-0x1019802a9bc0000 connected 2024-12-02T15:46:16,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:16,892 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:16,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:16,916 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82, hbase.cluster.distributed=false 2024-12-02T15:46:16,946 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T15:46:16,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42629 2024-12-02T15:46:16,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42629 2024-12-02T15:46:16,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42629 2024-12-02T15:46:16,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42629 2024-12-02T15:46:16,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42629 2024-12-02T15:46:17,052 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1c0b6c2739c:0 server-side Connection retries=45 2024-12-02T15:46:17,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,054 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T15:46:17,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,054 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T15:46:17,056 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T15:46:17,058 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T15:46:17,059 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40775 2024-12-02T15:46:17,060 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40775 connecting to ZooKeeper ensemble=127.0.0.1:53141 2024-12-02T15:46:17,061 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,063 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407750x0, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T15:46:17,077 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40775-0x1019802a9bc0001 connected 2024-12-02T15:46:17,078 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:17,082 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T15:46:17,091 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T15:46:17,094 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T15:46:17,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T15:46:17,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40775 2024-12-02T15:46:17,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40775 2024-12-02T15:46:17,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40775 2024-12-02T15:46:17,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40775 2024-12-02T15:46:17,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40775 2024-12-02T15:46:17,118 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1c0b6c2739c:0 server-side Connection retries=45 2024-12-02T15:46:17,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,119 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T15:46:17,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T15:46:17,120 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T15:46:17,120 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T15:46:17,121 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39549 2024-12-02T15:46:17,122 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39549 connecting to ZooKeeper ensemble=127.0.0.1:53141 2024-12-02T15:46:17,123 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,125 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395490x0, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T15:46:17,144 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39549-0x1019802a9bc0002 connected 2024-12-02T15:46:17,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:17,145 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T15:46:17,146 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T15:46:17,147 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T15:46:17,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T15:46:17,150 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39549 2024-12-02T15:46:17,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39549 2024-12-02T15:46:17,152 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39549 2024-12-02T15:46:17,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39549 2024-12-02T15:46:17,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39549 2024-12-02T15:46:17,174 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e1c0b6c2739c:0 server-side Connection retries=45 2024-12-02T15:46:17,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,175 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T15:46:17,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T15:46:17,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T15:46:17,175 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T15:46:17,175 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T15:46:17,176 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44999 2024-12-02T15:46:17,178 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44999 connecting to ZooKeeper ensemble=127.0.0.1:53141 2024-12-02T15:46:17,180 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,183 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:449990x0, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T15:46:17,202 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:449990x0, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:17,202 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44999-0x1019802a9bc0003 connected 2024-12-02T15:46:17,203 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T15:46:17,205 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T15:46:17,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T15:46:17,209 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T15:46:17,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44999 2024-12-02T15:46:17,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44999 2024-12-02T15:46:17,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44999 2024-12-02T15:46:17,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44999 2024-12-02T15:46:17,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44999 2024-12-02T15:46:17,228 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e1c0b6c2739c:42629 2024-12-02T15:46:17,229 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:17,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,245 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T15:46:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T15:46:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T15:46:17,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,275 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T15:46:17,276 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e1c0b6c2739c,42629,1733154376263 from backup master directory 2024-12-02T15:46:17,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:17,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T15:46:17,286 WARN [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T15:46:17,286 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:17,289 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T15:46:17,291 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T15:46:17,350 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/hbase.id] with ID: 118ff452-4204-444e-bcdf-b6b51bcf4b4f 2024-12-02T15:46:17,350 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/.tmp/hbase.id 2024-12-02T15:46:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741826_1002 (size=42) 2024-12-02T15:46:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741826_1002 (size=42) 2024-12-02T15:46:17,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741826_1002 (size=42) 2024-12-02T15:46:17,366 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/.tmp/hbase.id]:[hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/hbase.id] 2024-12-02T15:46:17,408 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T15:46:17,412 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T15:46:17,428 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-02T15:46:17,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:17,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741827_1003 (size=196) 2024-12-02T15:46:17,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741827_1003 (size=196) 2024-12-02T15:46:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741827_1003 (size=196) 2024-12-02T15:46:17,480 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T15:46:17,483 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T15:46:17,488 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:17,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741828_1004 (size=1189) 2024-12-02T15:46:17,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741828_1004 (size=1189) 2024-12-02T15:46:17,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741828_1004 (size=1189) 2024-12-02T15:46:17,540 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store 2024-12-02T15:46:17,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741829_1005 (size=34) 2024-12-02T15:46:17,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741829_1005 (size=34) 2024-12-02T15:46:17,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741829_1005 (size=34) 2024-12-02T15:46:17,567 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T15:46:17,570 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:17,572 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T15:46:17,572 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:17,572 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:17,574 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T15:46:17,574 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:17,574 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:17,575 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733154377571Disabling compacts and flushes for region at 1733154377572 (+1 ms)Disabling writes for close at 1733154377574 (+2 ms)Writing region close event to WAL at 1733154377574Closed at 1733154377574 2024-12-02T15:46:17,577 WARN [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/.initializing 2024-12-02T15:46:17,577 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/WALs/e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:17,584 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:17,601 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1c0b6c2739c%2C42629%2C1733154376263, suffix=, logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/WALs/e1c0b6c2739c,42629,1733154376263, archiveDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/oldWALs, maxLogs=10 2024-12-02T15:46:17,632 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/WALs/e1c0b6c2739c,42629,1733154376263/e1c0b6c2739c%2C42629%2C1733154376263.1733154377606, exclude list is [], retry=0 2024-12-02T15:46:17,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:17,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:17,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:17,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:17,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T15:46:17,683 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-02T15:46:17,740 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/WALs/e1c0b6c2739c,42629,1733154376263/e1c0b6c2739c%2C42629%2C1733154376263.1733154377606 2024-12-02T15:46:17,741 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:17,741 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:17,742 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:17,745 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,746 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T15:46:17,812 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:17,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:17,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T15:46:17,818 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:17,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:17,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T15:46:17,822 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:17,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:17,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T15:46:17,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:17,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:17,826 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,829 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,830 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,834 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,835 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,838 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T15:46:17,842 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T15:46:17,846 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:17,847 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71506529, jitterRate=0.06553031504154205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T15:46:17,852 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733154377757Initializing all the Stores at 1733154377760 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154377760Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154377761 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154377761Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154377761Cleaning up temporary data from old regions at 1733154377835 (+74 ms)Region opened successfully at 1733154377852 (+17 ms) 2024-12-02T15:46:17,853 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T15:46:17,887 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1edf82b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1c0b6c2739c/172.17.0.3:0 2024-12-02T15:46:17,916 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T15:46:17,925 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T15:46:17,925 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T15:46:17,928 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T15:46:17,929 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T15:46:17,934 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T15:46:17,934 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T15:46:17,961 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T15:46:17,969 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T15:46:18,006 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T15:46:18,010 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T15:46:18,013 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T15:46:18,026 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T15:46:18,029 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T15:46:18,033 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T15:46:18,043 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T15:46:18,045 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T15:46:18,051 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T15:46:18,073 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T15:46:18,081 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,096 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e1c0b6c2739c,42629,1733154376263, sessionid=0x1019802a9bc0000, setting cluster-up flag (Was=false) 2024-12-02T15:46:18,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,143 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T15:46:18,145 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:18,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:18,189 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T15:46:18,191 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:18,198 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T15:46:18,216 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(746): ClusterId : 118ff452-4204-444e-bcdf-b6b51bcf4b4f 2024-12-02T15:46:18,216 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(746): ClusterId : 118ff452-4204-444e-bcdf-b6b51bcf4b4f 2024-12-02T15:46:18,216 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(746): ClusterId : 118ff452-4204-444e-bcdf-b6b51bcf4b4f 2024-12-02T15:46:18,218 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T15:46:18,218 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T15:46:18,218 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T15:46:18,271 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T15:46:18,279 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T15:46:18,283 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T15:46:18,283 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T15:46:18,283 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T15:46:18,283 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T15:46:18,283 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T15:46:18,283 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T15:46:18,286 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T15:46:18,292 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e1c0b6c2739c,42629,1733154376263 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T15:46:18,325 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T15:46:18,325 DEBUG [RS:1;e1c0b6c2739c:39549 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac91acb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1c0b6c2739c/172.17.0.3:0 2024-12-02T15:46:18,325 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T15:46:18,326 DEBUG [RS:2;e1c0b6c2739c:44999 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c1d08b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1c0b6c2739c/172.17.0.3:0 2024-12-02T15:46:18,326 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e1c0b6c2739c:0, corePoolSize=5, maxPoolSize=5 2024-12-02T15:46:18,326 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e1c0b6c2739c:0, corePoolSize=5, maxPoolSize=5 2024-12-02T15:46:18,326 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T15:46:18,326 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e1c0b6c2739c:0, corePoolSize=5, maxPoolSize=5 2024-12-02T15:46:18,327 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e1c0b6c2739c:0, corePoolSize=5, maxPoolSize=5 2024-12-02T15:46:18,327 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e1c0b6c2739c:0, corePoolSize=10, maxPoolSize=10 2024-12-02T15:46:18,327 DEBUG [RS:0;e1c0b6c2739c:40775 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776a16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e1c0b6c2739c/172.17.0.3:0 2024-12-02T15:46:18,327 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,327 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e1c0b6c2739c:0, corePoolSize=2, maxPoolSize=2 2024-12-02T15:46:18,327 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,341 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T15:46:18,341 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T15:46:18,342 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e1c0b6c2739c:40775 2024-12-02T15:46:18,343 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e1c0b6c2739c:44999 2024-12-02T15:46:18,343 DEBUG [RS:1;e1c0b6c2739c:39549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e1c0b6c2739c:39549 2024-12-02T15:46:18,346 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T15:46:18,346 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T15:46:18,346 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T15:46:18,347 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T15:46:18,347 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T15:46:18,347 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T15:46:18,347 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T15:46:18,347 DEBUG [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T15:46:18,347 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T15:46:18,349 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1c0b6c2739c,42629,1733154376263 with port=40775, startcode=1733154377022 2024-12-02T15:46:18,349 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1c0b6c2739c,42629,1733154376263 with port=39549, startcode=1733154377118 2024-12-02T15:46:18,349 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(2659): reportForDuty to master=e1c0b6c2739c,42629,1733154376263 with port=44999, startcode=1733154377173 2024-12-02T15:46:18,350 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:18,350 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T15:46:18,354 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733154408354 2024-12-02T15:46:18,356 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T15:46:18,357 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T15:46:18,361 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T15:46:18,362 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T15:46:18,362 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T15:46:18,362 DEBUG [RS:2;e1c0b6c2739c:44999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T15:46:18,362 DEBUG [RS:0;e1c0b6c2739c:40775 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T15:46:18,362 DEBUG [RS:1;e1c0b6c2739c:39549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T15:46:18,362 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T15:46:18,367 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,373 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T15:46:18,374 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T15:46:18,374 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T15:46:18,383 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T15:46:18,403 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T15:46:18,405 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e1c0b6c2739c:0:becomeActiveMaster-HFileCleaner.large.0-1733154378405,5,FailOnTimeoutGroup] 2024-12-02T15:46:18,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741831_1007 (size=1321) 2024-12-02T15:46:18,406 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e1c0b6c2739c:0:becomeActiveMaster-HFileCleaner.small.0-1733154378406,5,FailOnTimeoutGroup] 2024-12-02T15:46:18,407 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,407 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T15:46:18,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741831_1007 (size=1321) 2024-12-02T15:46:18,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741831_1007 (size=1321) 2024-12-02T15:46:18,408 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,408 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,409 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T15:46:18,409 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82 2024-12-02T15:46:18,427 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39617, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T15:46:18,427 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T15:46:18,427 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44115, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T15:46:18,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741832_1008 (size=32) 2024-12-02T15:46:18,434 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741832_1008 (size=32) 2024-12-02T15:46:18,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741832_1008 (size=32) 2024-12-02T15:46:18,437 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(517): Registering regionserver=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,438 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:18,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T15:46:18,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T15:46:18,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:18,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:18,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T15:46:18,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T15:46:18,451 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:18,452 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:18,452 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(517): Registering regionserver=e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:18,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:18,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T15:46:18,456 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82 2024-12-02T15:46:18,456 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33469 2024-12-02T15:46:18,456 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T15:46:18,456 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T15:46:18,457 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:18,458 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:18,459 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(517): Registering regionserver=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:18,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:18,459 DEBUG [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82 2024-12-02T15:46:18,460 DEBUG [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33469 2024-12-02T15:46:18,460 DEBUG [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T15:46:18,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T15:46:18,463 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82 2024-12-02T15:46:18,463 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33469 2024-12-02T15:46:18,463 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T15:46:18,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T15:46:18,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:18,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:18,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T15:46:18,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740 2024-12-02T15:46:18,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740 2024-12-02T15:46:18,471 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T15:46:18,472 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T15:46:18,473 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T15:46:18,476 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T15:46:18,482 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:18,484 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67031266, jitterRate=-0.0011563003063201904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T15:46:18,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733154378438Initializing all the Stores at 1733154378441 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154378441Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154378442 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154378442Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154378442Cleaning up temporary data from old regions at 1733154378472 (+30 ms)Region opened successfully at 1733154378487 (+15 ms) 2024-12-02T15:46:18,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T15:46:18,488 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T15:46:18,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T15:46:18,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T15:46:18,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T15:46:18,491 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T15:46:18,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733154378488Disabling compacts and flushes for region at 1733154378488Disabling writes for close at 1733154378489 (+1 ms)Writing region close event to WAL at 1733154378490 (+1 ms)Closed at 1733154378491 (+1 ms) 2024-12-02T15:46:18,494 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T15:46:18,495 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T15:46:18,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T15:46:18,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T15:46:18,515 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T15:46:18,518 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T15:46:18,571 DEBUG [RS:2;e1c0b6c2739c:44999 {}] zookeeper.ZKUtil(111): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,571 WARN [RS:2;e1c0b6c2739c:44999 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T15:46:18,571 DEBUG [RS:0;e1c0b6c2739c:40775 {}] zookeeper.ZKUtil(111): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:18,571 DEBUG [RS:1;e1c0b6c2739c:39549 {}] zookeeper.ZKUtil(111): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:18,571 WARN [RS:1;e1c0b6c2739c:39549 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T15:46:18,571 WARN [RS:0;e1c0b6c2739c:40775 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T15:46:18,571 INFO [RS:2;e1c0b6c2739c:44999 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:18,572 INFO [RS:0;e1c0b6c2739c:40775 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:18,572 INFO [RS:1;e1c0b6c2739c:39549 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:18,572 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,572 DEBUG [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:18,572 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:18,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1c0b6c2739c,39549,1733154377118] 2024-12-02T15:46:18,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1c0b6c2739c,44999,1733154377173] 2024-12-02T15:46:18,573 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e1c0b6c2739c,40775,1733154377022] 2024-12-02T15:46:18,599 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T15:46:18,599 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T15:46:18,599 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T15:46:18,613 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T15:46:18,613 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T15:46:18,613 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T15:46:18,620 INFO [RS:1;e1c0b6c2739c:39549 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T15:46:18,620 INFO [RS:2;e1c0b6c2739c:44999 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T15:46:18,620 INFO [RS:0;e1c0b6c2739c:40775 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T15:46:18,620 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,620 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,620 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,621 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T15:46:18,621 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T15:46:18,621 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T15:46:18,628 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T15:46:18,628 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T15:46:18,628 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T15:46:18,630 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,630 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,630 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,631 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0, corePoolSize=2, maxPoolSize=2 2024-12-02T15:46:18,631 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,631 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0, corePoolSize=2, maxPoolSize=2 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0, corePoolSize=2, maxPoolSize=2 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1c0b6c2739c:0, corePoolSize=3, maxPoolSize=3 2024-12-02T15:46:18,632 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:2;e1c0b6c2739c:44999 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1c0b6c2739c:0, corePoolSize=3, maxPoolSize=3 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,632 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,633 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1c0b6c2739c:0, corePoolSize=3, maxPoolSize=3 2024-12-02T15:46:18,633 DEBUG [RS:0;e1c0b6c2739c:40775 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1c0b6c2739c:0, corePoolSize=3, maxPoolSize=3 2024-12-02T15:46:18,633 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e1c0b6c2739c:0, corePoolSize=1, maxPoolSize=1 2024-12-02T15:46:18,633 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e1c0b6c2739c:0, corePoolSize=3, maxPoolSize=3 2024-12-02T15:46:18,633 DEBUG [RS:1;e1c0b6c2739c:39549 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e1c0b6c2739c:0, corePoolSize=3, maxPoolSize=3 2024-12-02T15:46:18,638 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,638 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,638 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,638 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,639 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,639 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,44999,1733154377173-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T15:46:18,642 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,40775,1733154377022-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,643 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,39549,1733154377118-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T15:46:18,661 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T15:46:18,663 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,44999,1733154377173-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,663 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,664 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.Replication(171): e1c0b6c2739c,44999,1733154377173 started 2024-12-02T15:46:18,664 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T15:46:18,664 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,40775,1733154377022-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,664 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,665 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.Replication(171): e1c0b6c2739c,40775,1733154377022 started 2024-12-02T15:46:18,666 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T15:46:18,666 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,39549,1733154377118-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,666 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,667 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.Replication(171): e1c0b6c2739c,39549,1733154377118 started 2024-12-02T15:46:18,669 WARN [e1c0b6c2739c:42629 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T15:46:18,683 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,684 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1482): Serving as e1c0b6c2739c,44999,1733154377173, RpcServer on e1c0b6c2739c/172.17.0.3:44999, sessionid=0x1019802a9bc0003 2024-12-02T15:46:18,684 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T15:46:18,685 DEBUG [RS:2;e1c0b6c2739c:44999 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,685 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1c0b6c2739c,44999,1733154377173' 2024-12-02T15:46:18,685 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T15:46:18,686 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T15:46:18,686 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,687 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1482): Serving as e1c0b6c2739c,40775,1733154377022, RpcServer on e1c0b6c2739c/172.17.0.3:40775, sessionid=0x1019802a9bc0001 2024-12-02T15:46:18,687 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T15:46:18,687 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T15:46:18,687 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T15:46:18,687 DEBUG [RS:0;e1c0b6c2739c:40775 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:18,687 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1c0b6c2739c,40775,1733154377022' 2024-12-02T15:46:18,687 DEBUG [RS:2;e1c0b6c2739c:44999 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,687 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T15:46:18,687 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1c0b6c2739c,44999,1733154377173' 2024-12-02T15:46:18,687 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T15:46:18,688 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:18,688 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T15:46:18,688 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(1482): Serving as e1c0b6c2739c,39549,1733154377118, RpcServer on e1c0b6c2739c/172.17.0.3:39549, sessionid=0x1019802a9bc0002 2024-12-02T15:46:18,688 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T15:46:18,688 DEBUG [RS:1;e1c0b6c2739c:39549 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:18,688 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1c0b6c2739c,39549,1733154377118' 2024-12-02T15:46:18,688 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T15:46:18,689 DEBUG [RS:2;e1c0b6c2739c:44999 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T15:46:18,689 INFO [RS:2;e1c0b6c2739c:44999 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T15:46:18,689 INFO [RS:2;e1c0b6c2739c:44999 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T15:46:18,689 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T15:46:18,689 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T15:46:18,690 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T15:46:18,690 DEBUG [RS:1;e1c0b6c2739c:39549 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:18,690 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1c0b6c2739c,39549,1733154377118' 2024-12-02T15:46:18,690 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T15:46:18,690 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T15:46:18,690 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T15:46:18,691 DEBUG [RS:1;e1c0b6c2739c:39549 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T15:46:18,691 INFO [RS:1;e1c0b6c2739c:39549 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T15:46:18,691 INFO [RS:1;e1c0b6c2739c:39549 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T15:46:18,691 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T15:46:18,691 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T15:46:18,691 DEBUG [RS:0;e1c0b6c2739c:40775 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:18,691 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e1c0b6c2739c,40775,1733154377022' 2024-12-02T15:46:18,691 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T15:46:18,692 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T15:46:18,692 DEBUG [RS:0;e1c0b6c2739c:40775 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T15:46:18,693 INFO [RS:0;e1c0b6c2739c:40775 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T15:46:18,693 INFO [RS:0;e1c0b6c2739c:40775 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T15:46:18,794 INFO [RS:2;e1c0b6c2739c:44999 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:18,794 INFO [RS:0;e1c0b6c2739c:40775 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:18,794 INFO [RS:1;e1c0b6c2739c:39549 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:18,797 INFO [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1c0b6c2739c%2C40775%2C1733154377022, suffix=, logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022, archiveDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs, maxLogs=32 2024-12-02T15:46:18,797 INFO [RS:2;e1c0b6c2739c:44999 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1c0b6c2739c%2C44999%2C1733154377173, suffix=, logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173, archiveDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs, maxLogs=32 2024-12-02T15:46:18,797 INFO [RS:1;e1c0b6c2739c:39549 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1c0b6c2739c%2C39549%2C1733154377118, suffix=, logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,39549,1733154377118, archiveDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs, maxLogs=32 2024-12-02T15:46:18,817 DEBUG [RS:2;e1c0b6c2739c:44999 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, exclude list is [], retry=0 2024-12-02T15:46:18,818 DEBUG [RS:1;e1c0b6c2739c:39549 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,39549,1733154377118/e1c0b6c2739c%2C39549%2C1733154377118.1733154378800, exclude list is [], retry=0 2024-12-02T15:46:18,818 DEBUG [RS:0;e1c0b6c2739c:40775 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022/e1c0b6c2739c%2C40775%2C1733154377022.1733154378800, exclude list is [], retry=0 2024-12-02T15:46:18,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:18,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:18,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:18,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:18,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:18,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:18,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:18,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:18,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:18,855 INFO [RS:1;e1c0b6c2739c:39549 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,39549,1733154377118/e1c0b6c2739c%2C39549%2C1733154377118.1733154378800 2024-12-02T15:46:18,857 INFO [RS:2;e1c0b6c2739c:44999 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 2024-12-02T15:46:18,857 DEBUG [RS:1;e1c0b6c2739c:39549 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:18,858 DEBUG [RS:2;e1c0b6c2739c:44999 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:18,862 INFO [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022/e1c0b6c2739c%2C40775%2C1733154377022.1733154378800 2024-12-02T15:46:18,864 DEBUG [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:18,922 DEBUG [e1c0b6c2739c:42629 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-02T15:46:18,929 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(204): Hosts are {e1c0b6c2739c=0} racks are {/default-rack=0} 2024-12-02T15:46:18,934 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T15:46:18,935 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T15:46:18,935 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T15:46:18,935 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T15:46:18,935 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T15:46:18,935 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T15:46:18,935 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T15:46:18,935 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T15:46:18,935 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T15:46:18,935 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T15:46:18,941 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:18,946 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1c0b6c2739c,44999,1733154377173, state=OPENING 2024-12-02T15:46:18,990 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T15:46:19,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:19,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:19,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:19,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:19,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,004 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T15:46:19,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e1c0b6c2739c,44999,1733154377173}] 2024-12-02T15:46:19,009 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,186 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T15:46:19,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51699, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T15:46:19,205 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T15:46:19,206 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:19,207 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T15:46:19,210 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1c0b6c2739c%2C44999%2C1733154377173.meta, suffix=.meta, logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173, archiveDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs, maxLogs=32 2024-12-02T15:46:19,230 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, exclude list is [], retry=0 2024-12-02T15:46:19,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:19,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:19,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:19,275 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta 2024-12-02T15:46:19,276 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:19,276 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:19,278 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T15:46:19,280 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T15:46:19,284 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T15:46:19,289 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T15:46:19,289 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:19,289 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T15:46:19,290 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T15:46:19,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T15:46:19,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T15:46:19,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:19,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:19,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T15:46:19,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T15:46:19,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:19,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:19,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T15:46:19,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T15:46:19,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:19,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:19,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T15:46:19,305 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T15:46:19,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:19,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:19,306 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T15:46:19,308 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740 2024-12-02T15:46:19,311 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740 2024-12-02T15:46:19,313 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T15:46:19,313 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T15:46:19,314 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T15:46:19,317 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T15:46:19,318 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62972393, jitterRate=-0.06163822114467621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T15:46:19,318 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T15:46:19,320 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733154379290Writing region info on filesystem at 1733154379290Initializing all the Stores at 1733154379292 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154379292Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154379294 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154379294Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154379294Cleaning up temporary data from old regions at 1733154379313 (+19 ms)Running coprocessor post-open hooks at 1733154379318 (+5 ms)Region opened successfully at 1733154379320 (+2 ms) 2024-12-02T15:46:19,329 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733154379177 2024-12-02T15:46:19,343 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T15:46:19,344 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T15:46:19,348 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:19,351 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1c0b6c2739c,44999,1733154377173, state=OPEN 2024-12-02T15:46:19,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:19,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:19,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:19,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:19,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:19,360 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:19,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T15:46:19,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e1c0b6c2739c,44999,1733154377173 in 355 msec 2024-12-02T15:46:19,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T15:46:19,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 868 msec 2024-12-02T15:46:19,382 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T15:46:19,382 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T15:46:19,410 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:19,412 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1] 2024-12-02T15:46:19,438 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T15:46:19,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T15:46:19,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2310 sec 2024-12-02T15:46:19,463 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733154379463, completionTime=-1 2024-12-02T15:46:19,466 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-02T15:46:19,467 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T15:46:19,497 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-02T15:46:19,497 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733154439497 2024-12-02T15:46:19,498 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733154499498 2024-12-02T15:46:19,498 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-02T15:46:19,499 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-02T15:46:19,508 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,42629,1733154376263-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,508 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,42629,1733154376263-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,508 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,42629,1733154376263-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,510 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e1c0b6c2739c:42629, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,510 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,513 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,539 DEBUG [master/e1c0b6c2739c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T15:46:19,560 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.273sec 2024-12-02T15:46:19,565 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T15:46:19,568 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T15:46:19,569 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T15:46:19,569 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T15:46:19,570 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T15:46:19,570 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,42629,1733154376263-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T15:46:19,571 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,42629,1733154376263-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T15:46:19,575 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T15:46:19,576 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T15:46:19,576 INFO [master/e1c0b6c2739c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e1c0b6c2739c,42629,1733154376263-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:19,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600d4a87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T15:46:19,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e1c0b6c2739c,42629,-1 for getting cluster id 2024-12-02T15:46:19,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T15:46:19,641 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '118ff452-4204-444e-bcdf-b6b51bcf4b4f' 2024-12-02T15:46:19,643 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T15:46:19,644 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "118ff452-4204-444e-bcdf-b6b51bcf4b4f" 2024-12-02T15:46:19,644 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5175daac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T15:46:19,644 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e1c0b6c2739c,42629,-1] 2024-12-02T15:46:19,647 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T15:46:19,649 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:19,651 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T15:46:19,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be8dbe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T15:46:19,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:19,663 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1] 2024-12-02T15:46:19,664 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T15:46:19,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T15:46:19,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:19,690 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33469/hbase 2024-12-02T15:46:19,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=10261 2024-12-02T15:46:19,728 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:19,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:19,733 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:19,737 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34237648, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-34237648, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:19,752 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34237648/hregion-34237648.1733154379738, exclude list is [], retry=0 2024-12-02T15:46:19,756 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:19,756 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:19,759 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:19,792 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34237648/hregion-34237648.1733154379738 2024-12-02T15:46:19,792 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:19,792 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6a1b9c37a97db6791c9844e822a6e232, NAME => 'testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:19,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741838_1014 (size=64) 2024-12-02T15:46:19,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741838_1014 (size=64) 2024-12-02T15:46:19,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741838_1014 (size=64) 2024-12-02T15:46:19,808 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:19,810 INFO [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,812 INFO [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a1b9c37a97db6791c9844e822a6e232 columnFamilyName a 2024-12-02T15:46:19,813 DEBUG [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:19,814 INFO [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] regionserver.HStore(327): Store=6a1b9c37a97db6791c9844e822a6e232/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:19,814 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,815 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,816 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,816 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,816 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,819 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:19,823 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:19,824 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a1b9c37a97db6791c9844e822a6e232; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65453712, jitterRate=-0.024663686752319336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:19,825 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a1b9c37a97db6791c9844e822a6e232: Writing region info on filesystem at 1733154379808Initializing all the Stores at 1733154379809 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154379810 (+1 ms)Cleaning up temporary data from old regions at 1733154379816 (+6 ms)Region opened successfully at 1733154379825 (+9 ms) 2024-12-02T15:46:19,826 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a1b9c37a97db6791c9844e822a6e232, disabling compactions & flushes 2024-12-02T15:46:19,826 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232. 2024-12-02T15:46:19,826 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232. 2024-12-02T15:46:19,826 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232. after waiting 0 ms 2024-12-02T15:46:19,826 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232. 2024-12-02T15:46:19,826 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232. 2024-12-02T15:46:19,826 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a1b9c37a97db6791c9844e822a6e232: Waiting for close lock at 1733154379825Disabling compacts and flushes for region at 1733154379826 (+1 ms)Disabling writes for close at 1733154379826Writing region close event to WAL at 1733154379826Closed at 1733154379826 2024-12-02T15:46:19,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741837_1013 (size=175) 2024-12-02T15:46:19,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741837_1013 (size=175) 2024-12-02T15:46:19,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741837_1013 (size=175) 2024-12-02T15:46:19,841 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:19,841 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34237648:(num 1733154379738) 2024-12-02T15:46:19,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T15:46:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741839_1015 (size=469) 2024-12-02T15:46:19,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741839_1015 (size=469) 2024-12-02T15:46:19,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741839_1015 (size=469) 2024-12-02T15:46:19,883 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T15:46:19,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741840_1016 (size=322) 2024-12-02T15:46:19,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741840_1016 (size=322) 2024-12-02T15:46:19,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741840_1016 (size=322) 2024-12-02T15:46:19,938 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1, size=469 (469bytes) 2024-12-02T15:46:19,939 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-02T15:46:19,939 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-02T15:46:19,939 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1 2024-12-02T15:46:19,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1 after 3ms 2024-12-02T15:46:19,980 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:19,981 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1 took 44ms 2024-12-02T15:46:19,986 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1 so closing down 2024-12-02T15:46:19,986 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:19,989 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-02T15:46:20,013 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000001-wal-1.temp 2024-12-02T15:46:20,015 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741841_1017 (size=469) 2024-12-02T15:46:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741841_1017 (size=469) 2024-12-02T15:46:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741841_1017 (size=469) 2024-12-02T15:46:20,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:20,033 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002 2024-12-02T15:46:20,037 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-02T15:46:20,037 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1, journal: Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1, size=469 (469bytes) at 1733154379938Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1 so closing down at 1733154379986 (+48 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000001-wal-1.temp at 1733154380013 (+27 ms)3 split writer threads finished at 1733154380015 (+2 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733154380030 (+15 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002 at 1733154380033 (+3 ms)Processed 2 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733154380037 (+4 ms) 2024-12-02T15:46:20,051 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2, size=322 (322bytes) 2024-12-02T15:46:20,051 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2 2024-12-02T15:46:20,052 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2 after 1ms 2024-12-02T15:46:20,076 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:20,076 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2 took 26ms 2024-12-02T15:46:20,079 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2 so closing down 2024-12-02T15:46:20,079 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:20,082 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-02T15:46:20,106 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002-wal-2.temp 2024-12-02T15:46:20,106 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:20,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741842_1018 (size=322) 2024-12-02T15:46:20,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741842_1018 (size=322) 2024-12-02T15:46:20,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741842_1018 (size=322) 2024-12-02T15:46:20,116 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:20,145 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:20,148 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-02T15:46:20,149 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-02T15:46:20,149 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2, journal: Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2, size=322 (322bytes) at 1733154380051Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2 so closing down at 1733154380079 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002-wal-2.temp at 1733154380106 (+27 ms)3 split writer threads finished at 1733154380106Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733154380116 (+10 ms)Processed 1 edits across 1 Regions in 73 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733154380149 (+33 ms) 2024-12-02T15:46:20,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:20,152 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:20,166 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal.1733154380153, exclude list is [], retry=0 2024-12-02T15:46:20,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:20,171 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:20,171 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:20,195 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal.1733154380153 2024-12-02T15:46:20,196 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:20,196 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a1b9c37a97db6791c9844e822a6e232, NAME => 'testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:20,196 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:20,196 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,196 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,198 INFO [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,200 INFO [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a1b9c37a97db6791c9844e822a6e232 columnFamilyName a 2024-12-02T15:46:20,200 DEBUG [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:20,201 INFO [StoreOpener-6a1b9c37a97db6791c9844e822a6e232-1 {}] regionserver.HStore(327): Store=6a1b9c37a97db6791c9844e822a6e232/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:20,201 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,202 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,206 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002 2024-12-02T15:46:20,230 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:20,236 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002 2024-12-02T15:46:20,239 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a1b9c37a97db6791c9844e822a6e232 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-02T15:46:20,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/.tmp/a/74a4f2a6b4454e92a4b3b9fb35383f8a is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733154379842/Put/seqid=0 2024-12-02T15:46:20,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741844_1020 (size=5170) 2024-12-02T15:46:20,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741844_1020 (size=5170) 2024-12-02T15:46:20,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741844_1020 (size=5170) 2024-12-02T15:46:20,293 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/.tmp/a/74a4f2a6b4454e92a4b3b9fb35383f8a 2024-12-02T15:46:20,340 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/.tmp/a/74a4f2a6b4454e92a4b3b9fb35383f8a as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/a/74a4f2a6b4454e92a4b3b9fb35383f8a 2024-12-02T15:46:20,354 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/a/74a4f2a6b4454e92a4b3b9fb35383f8a, entries=2, sequenceid=2, filesize=5.0 K 2024-12-02T15:46:20,362 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6a1b9c37a97db6791c9844e822a6e232 in 120ms, sequenceid=2, compaction requested=false; wal=null 2024-12-02T15:46:20,364 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/0000000000000000002 2024-12-02T15:46:20,365 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,365 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a1b9c37a97db6791c9844e822a6e232 2024-12-02T15:46:20,372 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/6a1b9c37a97db6791c9844e822a6e232/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-02T15:46:20,374 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a1b9c37a97db6791c9844e822a6e232; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70207898, jitterRate=0.04617920517921448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:20,375 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a1b9c37a97db6791c9844e822a6e232: Writing region info on filesystem at 1733154380196Initializing all the Stores at 1733154380198 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154380198Obtaining lock to block concurrent updates at 1733154380239 (+41 ms)Preparing flush snapshotting stores in 6a1b9c37a97db6791c9844e822a6e232 at 1733154380239Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733154380242 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733154379729.6a1b9c37a97db6791c9844e822a6e232. at 1733154380242Flushing 6a1b9c37a97db6791c9844e822a6e232/a: creating writer at 1733154380243 (+1 ms)Flushing 6a1b9c37a97db6791c9844e822a6e232/a: appending metadata at 1733154380273 (+30 ms)Flushing 6a1b9c37a97db6791c9844e822a6e232/a: closing flushed file at 1733154380275 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79830647: reopening flushed file at 1733154380338 (+63 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 6a1b9c37a97db6791c9844e822a6e232 in 120ms, sequenceid=2, compaction requested=false; wal=null at 1733154380362 (+24 ms)Cleaning up temporary data from old regions at 1733154380365 (+3 ms)Region opened successfully at 1733154380375 (+10 ms) 2024-12-02T15:46:20,396 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 362) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33082 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:32966 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:54910 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33469/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:54982 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:45654 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:47804 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=10227 (was 10261) 2024-12-02T15:46:20,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=10225 2024-12-02T15:46:20,425 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:20,427 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:20,428 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:20,431 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79472009, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-79472009, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:20,443 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79472009/hregion-79472009.1733154380432, exclude list is [], retry=0 2024-12-02T15:46:20,447 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:20,447 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:20,447 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:20,470 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79472009/hregion-79472009.1733154380432 2024-12-02T15:46:20,471 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:20,471 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 584bd3d04d31f5c2ee248031873c0e91, NAME => 'testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:20,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741846_1022 (size=64) 2024-12-02T15:46:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741846_1022 (size=64) 2024-12-02T15:46:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741846_1022 (size=64) 2024-12-02T15:46:20,482 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:20,483 INFO [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,485 INFO [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 584bd3d04d31f5c2ee248031873c0e91 columnFamilyName a 2024-12-02T15:46:20,485 DEBUG [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:20,486 INFO [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] regionserver.HStore(327): Store=584bd3d04d31f5c2ee248031873c0e91/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:20,486 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,487 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,488 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,488 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,488 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,490 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:20,493 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:20,494 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 584bd3d04d31f5c2ee248031873c0e91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63518819, jitterRate=-0.05349583923816681}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:20,494 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 584bd3d04d31f5c2ee248031873c0e91: Writing region info on filesystem at 1733154380482Initializing all the Stores at 1733154380483 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154380483Cleaning up temporary data from old regions at 1733154380488 (+5 ms)Region opened successfully at 1733154380494 (+6 ms) 2024-12-02T15:46:20,494 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 584bd3d04d31f5c2ee248031873c0e91, disabling compactions & flushes 2024-12-02T15:46:20,494 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91. 2024-12-02T15:46:20,494 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91. 2024-12-02T15:46:20,494 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91. after waiting 0 ms 2024-12-02T15:46:20,494 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91. 2024-12-02T15:46:20,495 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91. 2024-12-02T15:46:20,495 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 584bd3d04d31f5c2ee248031873c0e91: Waiting for close lock at 1733154380494Disabling compacts and flushes for region at 1733154380494Disabling writes for close at 1733154380494Writing region close event to WAL at 1733154380495 (+1 ms)Closed at 1733154380495 2024-12-02T15:46:20,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741845_1021 (size=175) 2024-12-02T15:46:20,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741845_1021 (size=175) 2024-12-02T15:46:20,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741845_1021 (size=175) 2024-12-02T15:46:20,502 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:20,502 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79472009:(num 1733154380432) 2024-12-02T15:46:20,503 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T15:46:20,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741847_1023 (size=469) 2024-12-02T15:46:20,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741847_1023 (size=469) 2024-12-02T15:46:20,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741847_1023 (size=469) 2024-12-02T15:46:20,561 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-02T15:46:20,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741848_1024 (size=322) 2024-12-02T15:46:20,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741848_1024 (size=322) 2024-12-02T15:46:20,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741848_1024 (size=322) 2024-12-02T15:46:20,614 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2, size=322 (322bytes) 2024-12-02T15:46:20,614 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2 2024-12-02T15:46:20,615 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2 after 1ms 2024-12-02T15:46:20,641 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:20,641 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2 took 28ms 2024-12-02T15:46:20,644 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2 so closing down 2024-12-02T15:46:20,644 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:20,647 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-02T15:46:20,669 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002-wal-2.temp 2024-12-02T15:46:20,670 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:20,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741849_1025 (size=322) 2024-12-02T15:46:20,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741849_1025 (size=322) 2024-12-02T15:46:20,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741849_1025 (size=322) 2024-12-02T15:46:21,079 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:21,081 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 2024-12-02T15:46:21,082 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 440 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-02T15:46:21,082 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2, journal: Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2, size=322 (322bytes) at 1733154380614Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2 so closing down at 1733154380644 (+30 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002-wal-2.temp at 1733154380669 (+25 ms)3 split writer threads finished at 1733154380670 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733154381079 (+409 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 at 1733154381081 (+2 ms)Processed 1 edits across 1 Regions in 440 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733154381082 (+1 ms) 2024-12-02T15:46:21,101 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1, size=469 (469bytes) 2024-12-02T15:46:21,101 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1 2024-12-02T15:46:21,102 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1 after 0ms 2024-12-02T15:46:21,125 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:21,125 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1 took 25ms 2024-12-02T15:46:21,128 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1 so closing down 2024-12-02T15:46:21,128 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:21,130 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-02T15:46:21,154 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000001-wal-1.temp 2024-12-02T15:46:21,155 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:21,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741850_1026 (size=469) 2024-12-02T15:46:21,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741850_1026 (size=469) 2024-12-02T15:46:21,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741850_1026 (size=469) 2024-12-02T15:46:21,163 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:21,189 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:21,192 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002, length=322 2024-12-02T15:46:21,195 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 2024-12-02T15:46:21,195 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-02T15:46:21,195 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1, journal: Splitting hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1, size=469 (469bytes) at 1733154381101Finishing writing output for hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1 so closing down at 1733154381128 (+27 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000001-wal-1.temp at 1733154381155 (+27 ms)3 split writer threads finished at 1733154381155Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733154381164 (+9 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 at 1733154381195 (+31 ms)Processed 2 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733154381195 2024-12-02T15:46:21,195 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:21,197 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:21,208 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal.1733154381197, exclude list is [], retry=0 2024-12-02T15:46:21,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:21,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:21,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:21,237 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal.1733154381197 2024-12-02T15:46:21,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:21,238 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 584bd3d04d31f5c2ee248031873c0e91, NAME => 'testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:21,238 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:21,238 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,238 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,240 INFO [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,242 INFO [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 584bd3d04d31f5c2ee248031873c0e91 columnFamilyName a 2024-12-02T15:46:21,242 DEBUG [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:21,243 INFO [StoreOpener-584bd3d04d31f5c2ee248031873c0e91-1 {}] regionserver.HStore(327): Store=584bd3d04d31f5c2ee248031873c0e91/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:21,244 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,247 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,248 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 2024-12-02T15:46:21,273 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:21,275 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 2024-12-02T15:46:21,275 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 584bd3d04d31f5c2ee248031873c0e91 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-02T15:46:21,289 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/.tmp/a/96658c41faa54509a2daf952dc89d970 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733154380502/Put/seqid=0 2024-12-02T15:46:21,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741852_1028 (size=5170) 2024-12-02T15:46:21,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741852_1028 (size=5170) 2024-12-02T15:46:21,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741852_1028 (size=5170) 2024-12-02T15:46:21,297 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/.tmp/a/96658c41faa54509a2daf952dc89d970 2024-12-02T15:46:21,305 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/.tmp/a/96658c41faa54509a2daf952dc89d970 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/a/96658c41faa54509a2daf952dc89d970 2024-12-02T15:46:21,312 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/a/96658c41faa54509a2daf952dc89d970, entries=2, sequenceid=2, filesize=5.0 K 2024-12-02T15:46:21,313 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 584bd3d04d31f5c2ee248031873c0e91 in 38ms, sequenceid=2, compaction requested=false; wal=null 2024-12-02T15:46:21,313 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/0000000000000000002 2024-12-02T15:46:21,314 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,314 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,316 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 584bd3d04d31f5c2ee248031873c0e91 2024-12-02T15:46:21,319 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/584bd3d04d31f5c2ee248031873c0e91/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-02T15:46:21,320 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 584bd3d04d31f5c2ee248031873c0e91; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69770903, jitterRate=0.03966747224330902}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:21,320 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 584bd3d04d31f5c2ee248031873c0e91: Writing region info on filesystem at 1733154381238Initializing all the Stores at 1733154381240 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154381240Obtaining lock to block concurrent updates at 1733154381275 (+35 ms)Preparing flush snapshotting stores in 584bd3d04d31f5c2ee248031873c0e91 at 1733154381275Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733154381275Flushing stores of testReplayEditsWrittenIntoWAL,,1733154380426.584bd3d04d31f5c2ee248031873c0e91. at 1733154381275Flushing 584bd3d04d31f5c2ee248031873c0e91/a: creating writer at 1733154381275Flushing 584bd3d04d31f5c2ee248031873c0e91/a: appending metadata at 1733154381288 (+13 ms)Flushing 584bd3d04d31f5c2ee248031873c0e91/a: closing flushed file at 1733154381288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dc27203: reopening flushed file at 1733154381304 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 584bd3d04d31f5c2ee248031873c0e91 in 38ms, sequenceid=2, compaction requested=false; wal=null at 1733154381313 (+9 ms)Cleaning up temporary data from old regions at 1733154381314 (+1 ms)Region opened successfully at 1733154381320 (+6 ms) 2024-12-02T15:46:21,339 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:47870 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:32966 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:54910 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33148 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:55044 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:45654 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=10150 (was 10225) 2024-12-02T15:46:21,351 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=10149 2024-12-02T15:46:21,367 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:21,369 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:21,370 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:21,373 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24478674, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-24478674, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:21,384 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24478674/hregion-24478674.1733154381374, exclude list is [], retry=0 2024-12-02T15:46:21,387 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:21,387 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:21,388 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:21,414 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24478674/hregion-24478674.1733154381374 2024-12-02T15:46:21,414 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:21,414 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c5ef447d0db53c0bd1b31c17ba2d1bbe, NAME => 'testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:21,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741854_1030 (size=64) 2024-12-02T15:46:21,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741854_1030 (size=64) 2024-12-02T15:46:21,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741854_1030 (size=64) 2024-12-02T15:46:21,426 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:21,427 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,429 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5ef447d0db53c0bd1b31c17ba2d1bbe columnFamilyName a 2024-12-02T15:46:21,429 DEBUG [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:21,430 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(327): Store=c5ef447d0db53c0bd1b31c17ba2d1bbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:21,430 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,432 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5ef447d0db53c0bd1b31c17ba2d1bbe columnFamilyName b 2024-12-02T15:46:21,432 DEBUG [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:21,433 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(327): Store=c5ef447d0db53c0bd1b31c17ba2d1bbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:21,433 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,435 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5ef447d0db53c0bd1b31c17ba2d1bbe columnFamilyName c 2024-12-02T15:46:21,435 DEBUG [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:21,436 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(327): Store=c5ef447d0db53c0bd1b31c17ba2d1bbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:21,436 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,437 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,437 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,439 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,439 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,439 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:21,441 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:21,443 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:21,444 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c5ef447d0db53c0bd1b31c17ba2d1bbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65790390, jitterRate=-0.019646793603897095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:21,444 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c5ef447d0db53c0bd1b31c17ba2d1bbe: Writing region info on filesystem at 1733154381426Initializing all the Stores at 1733154381427 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154381427Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154381427Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154381427Cleaning up temporary data from old regions at 1733154381439 (+12 ms)Region opened successfully at 1733154381444 (+5 ms) 2024-12-02T15:46:21,444 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c5ef447d0db53c0bd1b31c17ba2d1bbe, disabling compactions & flushes 2024-12-02T15:46:21,444 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:21,445 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:21,445 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. after waiting 0 ms 2024-12-02T15:46:21,445 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:21,445 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:21,445 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c5ef447d0db53c0bd1b31c17ba2d1bbe: Waiting for close lock at 1733154381444Disabling compacts and flushes for region at 1733154381444Disabling writes for close at 1733154381445 (+1 ms)Writing region close event to WAL at 1733154381445Closed at 1733154381445 2024-12-02T15:46:21,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741853_1029 (size=175) 2024-12-02T15:46:21,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741853_1029 (size=175) 2024-12-02T15:46:21,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741853_1029 (size=175) 2024-12-02T15:46:21,452 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:21,453 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24478674:(num 1733154381374) 2024-12-02T15:46:21,453 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:21,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:21,466 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455, exclude list is [], retry=0 2024-12-02T15:46:21,469 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:21,469 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:21,470 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:21,494 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 2024-12-02T15:46:21,494 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:21,664 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455, size=0 (0bytes) 2024-12-02T15:46:21,664 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 might be still open, length is 0 2024-12-02T15:46:21,664 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 2024-12-02T15:46:21,665 WARN [IPC Server handler 4 on default port 33469 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-02T15:46:21,666 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 after 2ms 2024-12-02T15:46:24,441 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33166 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33166 dst: /127.0.0.1:38685 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38685 remote=/127.0.0.1:33166]. Total timeout mills is 60000, 57194 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:24,443 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:55086 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43105:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55086 dst: /127.0.0.1:43105 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:24,443 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:47900 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42367:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47900 dst: /127.0.0.1:42367 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:24,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741855_1032 (size=497775) 2024-12-02T15:46:24,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741855_1032 (size=497775) 2024-12-02T15:46:24,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741855_1032 (size=497775) 2024-12-02T15:46:24,802 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T15:46:24,851 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T15:46:25,668 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 after 4004ms 2024-12-02T15:46:25,703 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:25,704 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 took 4040ms 2024-12-02T15:46:25,709 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733154381455.temp 2024-12-02T15:46:25,738 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000000001-wal.1733154381455.temp 2024-12-02T15:46:25,797 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455; continuing. 2024-12-02T15:46:25,797 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 so closing down 2024-12-02T15:46:25,797 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:25,797 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741856_1033 (size=497783) 2024-12-02T15:46:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741856_1033 (size=497783) 2024-12-02T15:46:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741856_1033 (size=497783) 2024-12-02T15:46:25,805 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000000001-wal.1733154381455.temp (wrote 3002 edits, skipped 0 edits in 40 ms) 2024-12-02T15:46:25,807 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000000001-wal.1733154381455.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000003002 2024-12-02T15:46:25,807 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T15:46:25,807 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455, journal: Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455, size=0 (0bytes) at 1733154381664Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000000001-wal.1733154381455.temp at 1733154385738 (+4074 ms)Split 1024 edits, skipped 0 edits. at 1733154385759 (+21 ms)Split 2048 edits, skipped 0 edits. at 1733154385781 (+22 ms)Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 so closing down at 1733154385797 (+16 ms)3 split writer threads finished at 1733154385798 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000000001-wal.1733154381455.temp (wrote 3002 edits, skipped 0 edits in 40 ms) at 1733154385805 (+7 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000000001-wal.1733154381455.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000003002 at 1733154385807 (+2 ms)Processed 3002 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455, size=0, length=0, corrupted=false, cancelled=false at 1733154385807 2024-12-02T15:46:25,809 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154381455 2024-12-02T15:46:25,811 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000003002 2024-12-02T15:46:25,811 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:25,814 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:25,825 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154385814, exclude list is [], retry=0 2024-12-02T15:46:25,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:25,829 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:25,830 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:25,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154385814 2024-12-02T15:46:25,854 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:25,854 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:25,856 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:25,858 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5ef447d0db53c0bd1b31c17ba2d1bbe columnFamilyName a 2024-12-02T15:46:25,858 DEBUG [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:25,859 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(327): Store=c5ef447d0db53c0bd1b31c17ba2d1bbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:25,859 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:25,860 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5ef447d0db53c0bd1b31c17ba2d1bbe columnFamilyName b 2024-12-02T15:46:25,861 DEBUG [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:25,861 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(327): Store=c5ef447d0db53c0bd1b31c17ba2d1bbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:25,862 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:25,863 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5ef447d0db53c0bd1b31c17ba2d1bbe columnFamilyName c 2024-12-02T15:46:25,863 DEBUG [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:25,864 INFO [StoreOpener-c5ef447d0db53c0bd1b31c17ba2d1bbe-1 {}] regionserver.HStore(327): Store=c5ef447d0db53c0bd1b31c17ba2d1bbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:25,864 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:25,865 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:25,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:25,870 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000003002 2024-12-02T15:46:25,895 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:25,937 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T15:46:26,272 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c5ef447d0db53c0bd1b31c17ba2d1bbe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T15:46:26,303 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/a/79c58ad46b124f03b734e1e41219724b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733154381500/Put/seqid=0 2024-12-02T15:46:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741858_1035 (size=50463) 2024-12-02T15:46:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741858_1035 (size=50463) 2024-12-02T15:46:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741858_1035 (size=50463) 2024-12-02T15:46:26,312 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/a/79c58ad46b124f03b734e1e41219724b 2024-12-02T15:46:26,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/a/79c58ad46b124f03b734e1e41219724b as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/a/79c58ad46b124f03b734e1e41219724b 2024-12-02T15:46:26,328 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/a/79c58ad46b124f03b734e1e41219724b, entries=754, sequenceid=754, filesize=49.3 K 2024-12-02T15:46:26,329 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for c5ef447d0db53c0bd1b31c17ba2d1bbe in 56ms, sequenceid=754, compaction requested=false; wal=null 2024-12-02T15:46:26,341 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T15:46:26,341 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c5ef447d0db53c0bd1b31c17ba2d1bbe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T15:46:26,348 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/a/19de0ab5789748b3a12066071e807571 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733154381532/Put/seqid=0 2024-12-02T15:46:26,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741859_1036 (size=20072) 2024-12-02T15:46:26,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741859_1036 (size=20072) 2024-12-02T15:46:26,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741859_1036 (size=20072) 2024-12-02T15:46:26,358 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/a/19de0ab5789748b3a12066071e807571 2024-12-02T15:46:26,381 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/b/7de9ee1fc89344d5a75977f7903e2e66 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733154381557/Put/seqid=0 2024-12-02T15:46:26,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741860_1037 (size=35835) 2024-12-02T15:46:26,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741860_1037 (size=35835) 2024-12-02T15:46:26,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741860_1037 (size=35835) 2024-12-02T15:46:26,390 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/b/7de9ee1fc89344d5a75977f7903e2e66 2024-12-02T15:46:26,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/a/19de0ab5789748b3a12066071e807571 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/a/19de0ab5789748b3a12066071e807571 2024-12-02T15:46:26,406 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/a/19de0ab5789748b3a12066071e807571, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-02T15:46:26,408 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/b/7de9ee1fc89344d5a75977f7903e2e66 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/b/7de9ee1fc89344d5a75977f7903e2e66 2024-12-02T15:46:26,415 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/b/7de9ee1fc89344d5a75977f7903e2e66, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-02T15:46:26,415 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for c5ef447d0db53c0bd1b31c17ba2d1bbe in 74ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-02T15:46:26,435 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T15:46:26,436 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c5ef447d0db53c0bd1b31c17ba2d1bbe 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-02T15:46:26,443 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/b/aad1790d5c564df399d1e6a92aa2c2a1 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733154381572/Put/seqid=0 2024-12-02T15:46:26,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741861_1038 (size=35082) 2024-12-02T15:46:26,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741861_1038 (size=35082) 2024-12-02T15:46:26,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741861_1038 (size=35082) 2024-12-02T15:46:26,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/b/aad1790d5c564df399d1e6a92aa2c2a1 2024-12-02T15:46:26,480 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/c/07cd5d5ee47e461ab3e29f389cfeb859 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733154381597/Put/seqid=0 2024-12-02T15:46:26,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741862_1039 (size=20825) 2024-12-02T15:46:26,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741862_1039 (size=20825) 2024-12-02T15:46:26,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741862_1039 (size=20825) 2024-12-02T15:46:26,489 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/c/07cd5d5ee47e461ab3e29f389cfeb859 2024-12-02T15:46:26,498 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/b/aad1790d5c564df399d1e6a92aa2c2a1 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/b/aad1790d5c564df399d1e6a92aa2c2a1 2024-12-02T15:46:26,505 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/b/aad1790d5c564df399d1e6a92aa2c2a1, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-02T15:46:26,507 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/c/07cd5d5ee47e461ab3e29f389cfeb859 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/c/07cd5d5ee47e461ab3e29f389cfeb859 2024-12-02T15:46:26,514 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/c/07cd5d5ee47e461ab3e29f389cfeb859, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-02T15:46:26,515 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for c5ef447d0db53c0bd1b31c17ba2d1bbe in 79ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-02T15:46:26,524 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733154381633/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:26,527 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000003002 2024-12-02T15:46:26,527 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-02T15:46:26,528 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c5ef447d0db53c0bd1b31c17ba2d1bbe 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-02T15:46:26,535 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/c/d344a7299c9247f78b59b57221cba12e is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733154381604/Put/seqid=0 2024-12-02T15:46:26,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741863_1040 (size=50301) 2024-12-02T15:46:26,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741863_1040 (size=50301) 2024-12-02T15:46:26,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741863_1040 (size=50301) 2024-12-02T15:46:26,549 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/c/d344a7299c9247f78b59b57221cba12e 2024-12-02T15:46:26,555 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d344a7299c9247f78b59b57221cba12e 2024-12-02T15:46:26,557 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/.tmp/c/d344a7299c9247f78b59b57221cba12e as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/c/d344a7299c9247f78b59b57221cba12e 2024-12-02T15:46:26,564 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d344a7299c9247f78b59b57221cba12e 2024-12-02T15:46:26,564 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/c/d344a7299c9247f78b59b57221cba12e, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-02T15:46:26,564 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for c5ef447d0db53c0bd1b31c17ba2d1bbe in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-02T15:46:26,566 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/0000000000000003002 2024-12-02T15:46:26,567 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:26,567 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:26,568 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T15:46:26,571 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c5ef447d0db53c0bd1b31c17ba2d1bbe 2024-12-02T15:46:26,574 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenIntoWAL/c5ef447d0db53c0bd1b31c17ba2d1bbe/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-02T15:46:26,576 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c5ef447d0db53c0bd1b31c17ba2d1bbe; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71614876, jitterRate=0.06714481115341187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T15:46:26,576 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c5ef447d0db53c0bd1b31c17ba2d1bbe: Writing region info on filesystem at 1733154385854Initializing all the Stores at 1733154385856 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154385856Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154385856Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154385856Cleaning up temporary data from old regions at 1733154386567 (+711 ms)Region opened successfully at 1733154386576 (+9 ms) 2024-12-02T15:46:26,607 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T15:46:26,607 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:26,610 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T15:46:26,610 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T15:46:26,611 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T15:46:26,611 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:26,612 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-02T15:46:26,612 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:26,640 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c5ef447d0db53c0bd1b31c17ba2d1bbe, disabling compactions & flushes 2024-12-02T15:46:26,640 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:26,640 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:26,640 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. after waiting 0 ms 2024-12-02T15:46:26,640 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:26,642 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733154381368.c5ef447d0db53c0bd1b31c17ba2d1bbe. 2024-12-02T15:46:26,642 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c5ef447d0db53c0bd1b31c17ba2d1bbe: Waiting for close lock at 1733154386640Disabling compacts and flushes for region at 1733154386640Disabling writes for close at 1733154386640Writing region close event to WAL at 1733154386642 (+2 ms)Closed at 1733154386642 2024-12-02T15:46:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741857_1034 (size=175) 2024-12-02T15:46:26,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741857_1034 (size=175) 2024-12-02T15:46:26,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741857_1034 (size=175) 2024-12-02T15:46:26,649 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:26,649 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733154385814) 2024-12-02T15:46:26,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=395 (was 383) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34277 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34277 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-528606449_22 at /127.0.0.1:55102 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33469 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-528606449_22 at /127.0.0.1:33218 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-528606449_22 at /127.0.0.1:33226 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41261 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33469 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-528606449_22 at /127.0.0.1:47962 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41261 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=10055 (was 10149) 2024-12-02T15:46:26,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=395, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=10054 2024-12-02T15:46:26,687 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:26,690 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:26,691 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:26,694 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-17113719, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-17113719, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:26,706 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-17113719/hregion-17113719.1733154386695, exclude list is [], retry=0 2024-12-02T15:46:26,710 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:26,710 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:26,711 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:26,737 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-17113719/hregion-17113719.1733154386695 2024-12-02T15:46:26,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:26,737 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3b283966103e3011f6c81bd6071a8af4, NAME => 'test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:26,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741865_1042 (size=43) 2024-12-02T15:46:26,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741865_1042 (size=43) 2024-12-02T15:46:26,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741865_1042 (size=43) 2024-12-02T15:46:26,748 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:26,750 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,751 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b283966103e3011f6c81bd6071a8af4 columnFamilyName a 2024-12-02T15:46:26,752 DEBUG [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:26,752 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(327): Store=3b283966103e3011f6c81bd6071a8af4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:26,752 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,754 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b283966103e3011f6c81bd6071a8af4 columnFamilyName b 2024-12-02T15:46:26,754 DEBUG [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:26,755 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(327): Store=3b283966103e3011f6c81bd6071a8af4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:26,755 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,757 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b283966103e3011f6c81bd6071a8af4 columnFamilyName c 2024-12-02T15:46:26,757 DEBUG [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:26,757 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(327): Store=3b283966103e3011f6c81bd6071a8af4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:26,757 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,759 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,760 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,760 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,761 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:26,762 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:26,765 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:26,765 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3b283966103e3011f6c81bd6071a8af4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69997666, jitterRate=0.0430465042591095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:26,767 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3b283966103e3011f6c81bd6071a8af4: Writing region info on filesystem at 1733154386748Initializing all the Stores at 1733154386749 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154386750 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154386750Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154386750Cleaning up temporary data from old regions at 1733154386760 (+10 ms)Region opened successfully at 1733154386766 (+6 ms) 2024-12-02T15:46:26,767 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3b283966103e3011f6c81bd6071a8af4, disabling compactions & flushes 2024-12-02T15:46:26,767 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:26,767 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:26,767 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. after waiting 0 ms 2024-12-02T15:46:26,767 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:26,767 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:26,768 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3b283966103e3011f6c81bd6071a8af4: Waiting for close lock at 1733154386767Disabling compacts and flushes for region at 1733154386767Disabling writes for close at 1733154386767Writing region close event to WAL at 1733154386767Closed at 1733154386767 2024-12-02T15:46:26,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741864_1041 (size=175) 2024-12-02T15:46:26,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741864_1041 (size=175) 2024-12-02T15:46:26,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741864_1041 (size=175) 2024-12-02T15:46:26,776 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:26,776 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-17113719:(num 1733154386695) 2024-12-02T15:46:26,776 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:26,779 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:26,792 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779, exclude list is [], retry=0 2024-12-02T15:46:26,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:26,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:26,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:26,822 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 2024-12-02T15:46:26,823 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741866_1043 (size=371388) 2024-12-02T15:46:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741866_1043 (size=371388) 2024-12-02T15:46:26,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741866_1043 (size=371388) 2024-12-02T15:46:27,013 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779, size=362.7 K (371388bytes) 2024-12-02T15:46:27,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 2024-12-02T15:46:27,014 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 after 1ms 2024-12-02T15:46:27,045 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:27,047 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 took 35ms 2024-12-02T15:46:27,051 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733154386779.temp 2024-12-02T15:46:27,092 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 so closing down 2024-12-02T15:46:27,092 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:27,093 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000000001-wal.1733154386779.temp 2024-12-02T15:46:27,107 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:27,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741867_1044 (size=371388) 2024-12-02T15:46:27,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741867_1044 (size=371388) 2024-12-02T15:46:27,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741867_1044 (size=371388) 2024-12-02T15:46:27,115 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000000001-wal.1733154386779.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-12-02T15:46:27,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000000001-wal.1733154386779.temp to hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003000 2024-12-02T15:46:27,117 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-02T15:46:27,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779, journal: Splitting hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779, size=362.7 K (371388bytes) at 1733154387013Split 1024 edits, skipped 0 edits. at 1733154387063 (+50 ms)Split 2048 edits, skipped 0 edits. at 1733154387078 (+15 ms)Finishing writing output for hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 so closing down at 1733154387092 (+14 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000000001-wal.1733154386779.temp at 1733154387093 (+1 ms)3 split writer threads finished at 1733154387108 (+15 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000000001-wal.1733154386779.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1733154387115 (+7 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000000001-wal.1733154386779.temp to hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003000 at 1733154387117 (+2 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733154387117 2024-12-02T15:46:27,120 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154386779 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154386779 2024-12-02T15:46:27,122 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003000 2024-12-02T15:46:27,122 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:27,125 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:27,145 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126, exclude list is [], retry=0 2024-12-02T15:46:27,149 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:27,149 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:27,149 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:27,174 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 2024-12-02T15:46:27,175 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:27,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741868_1045 (size=371515) 2024-12-02T15:46:27,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741868_1045 (size=371515) 2024-12-02T15:46:27,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741868_1045 (size=371515) 2024-12-02T15:46:27,361 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126, size=362.8 K (371515bytes) 2024-12-02T15:46:27,361 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 2024-12-02T15:46:27,361 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 after 0ms 2024-12-02T15:46:27,390 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:27,392 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 took 32ms 2024-12-02T15:46:27,396 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733154387126.temp 2024-12-02T15:46:27,438 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 so closing down 2024-12-02T15:46:27,438 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:27,453 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003001-wal.1733154387126.temp 2024-12-02T15:46:27,461 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741869_1046 (size=371515) 2024-12-02T15:46:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741869_1046 (size=371515) 2024-12-02T15:46:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741869_1046 (size=371515) 2024-12-02T15:46:27,467 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003001-wal.1733154387126.temp (wrote 3000 edits, skipped 0 edits in 7 ms) 2024-12-02T15:46:27,469 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003001-wal.1733154387126.temp to hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000006000 2024-12-02T15:46:27,469 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-02T15:46:27,469 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126, journal: Splitting hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126, size=362.8 K (371515bytes) at 1733154387361Split 1024 edits, skipped 0 edits. at 1733154387403 (+42 ms)Split 2048 edits, skipped 0 edits. at 1733154387433 (+30 ms)Finishing writing output for hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 so closing down at 1733154387438 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003001-wal.1733154387126.temp at 1733154387453 (+15 ms)3 split writer threads finished at 1733154387461 (+8 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003001-wal.1733154387126.temp (wrote 3000 edits, skipped 0 edits in 7 ms) at 1733154387468 (+7 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003001-wal.1733154387126.temp to hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000006000 at 1733154387469 (+1 ms)Processed 3000 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733154387469 2024-12-02T15:46:27,472 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387126 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154387126 2024-12-02T15:46:27,473 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000006000 2024-12-02T15:46:27,473 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:27,475 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/test2727-manual,16010,1733154386686, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:27,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387475, exclude list is [], retry=0 2024-12-02T15:46:27,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:27,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:27,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:27,518 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733154386686/wal.1733154387475 2024-12-02T15:46:27,518 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:27,519 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3b283966103e3011f6c81bd6071a8af4, NAME => 'test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:27,519 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:27,519 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,519 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,521 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,522 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b283966103e3011f6c81bd6071a8af4 columnFamilyName a 2024-12-02T15:46:27,522 DEBUG [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:27,523 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(327): Store=3b283966103e3011f6c81bd6071a8af4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:27,523 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,524 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b283966103e3011f6c81bd6071a8af4 columnFamilyName b 2024-12-02T15:46:27,524 DEBUG [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:27,524 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(327): Store=3b283966103e3011f6c81bd6071a8af4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:27,525 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,526 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3b283966103e3011f6c81bd6071a8af4 columnFamilyName c 2024-12-02T15:46:27,526 DEBUG [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:27,527 INFO [StoreOpener-3b283966103e3011f6c81bd6071a8af4-1 {}] regionserver.HStore(327): Store=3b283966103e3011f6c81bd6071a8af4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:27,527 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,528 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:27,531 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003000 2024-12-02T15:46:27,566 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:27,611 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003000 2024-12-02T15:46:27,612 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000006000 2024-12-02T15:46:27,641 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:27,660 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000006000 2024-12-02T15:46:27,660 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3b283966103e3011f6c81bd6071a8af4 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-02T15:46:27,684 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/a/51c8006c87d94a34a14ce49d3606c8f3 is 41, key is test2727/a:100/1733154387182/Put/seqid=0 2024-12-02T15:46:27,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741871_1048 (size=84227) 2024-12-02T15:46:27,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741871_1048 (size=84227) 2024-12-02T15:46:27,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741871_1048 (size=84227) 2024-12-02T15:46:27,691 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/a/51c8006c87d94a34a14ce49d3606c8f3 2024-12-02T15:46:27,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/b/547620c534af4f5090aa1735d8631303 is 41, key is test2727/b:100/1733154387251/Put/seqid=0 2024-12-02T15:46:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741872_1049 (size=84609) 2024-12-02T15:46:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741872_1049 (size=84609) 2024-12-02T15:46:27,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741872_1049 (size=84609) 2024-12-02T15:46:28,125 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/b/547620c534af4f5090aa1735d8631303 2024-12-02T15:46:28,158 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/c/1dddd035d42c4782953d2bab3d468db9 is 41, key is test2727/c:100/1733154387294/Put/seqid=0 2024-12-02T15:46:28,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741873_1050 (size=84609) 2024-12-02T15:46:28,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741873_1050 (size=84609) 2024-12-02T15:46:28,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741873_1050 (size=84609) 2024-12-02T15:46:28,168 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/c/1dddd035d42c4782953d2bab3d468db9 2024-12-02T15:46:28,175 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/a/51c8006c87d94a34a14ce49d3606c8f3 as hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/a/51c8006c87d94a34a14ce49d3606c8f3 2024-12-02T15:46:28,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/a/51c8006c87d94a34a14ce49d3606c8f3, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-02T15:46:28,184 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/b/547620c534af4f5090aa1735d8631303 as hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/b/547620c534af4f5090aa1735d8631303 2024-12-02T15:46:28,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/b/547620c534af4f5090aa1735d8631303, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-02T15:46:28,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/.tmp/c/1dddd035d42c4782953d2bab3d468db9 as hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/c/1dddd035d42c4782953d2bab3d468db9 2024-12-02T15:46:28,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/c/1dddd035d42c4782953d2bab3d468db9, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-02T15:46:28,197 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 3b283966103e3011f6c81bd6071a8af4 in 537ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-02T15:46:28,198 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000003000 2024-12-02T15:46:28,199 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/0000000000000006000 2024-12-02T15:46:28,200 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:28,200 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:28,201 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:28,204 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3b283966103e3011f6c81bd6071a8af4 2024-12-02T15:46:28,207 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/test2727/3b283966103e3011f6c81bd6071a8af4/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-02T15:46:28,208 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3b283966103e3011f6c81bd6071a8af4; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67000111, jitterRate=-0.0016205459833145142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:28,210 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3b283966103e3011f6c81bd6071a8af4: Writing region info on filesystem at 1733154387519Initializing all the Stores at 1733154387520 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154387520Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154387520Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154387520Obtaining lock to block concurrent updates at 1733154387660 (+140 ms)Preparing flush snapshotting stores in 3b283966103e3011f6c81bd6071a8af4 at 1733154387660Finished memstore snapshotting test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733154387661 (+1 ms)Flushing stores of test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. at 1733154387661Flushing 3b283966103e3011f6c81bd6071a8af4/a: creating writer at 1733154387661Flushing 3b283966103e3011f6c81bd6071a8af4/a: appending metadata at 1733154387683 (+22 ms)Flushing 3b283966103e3011f6c81bd6071a8af4/a: closing flushed file at 1733154387684 (+1 ms)Flushing 3b283966103e3011f6c81bd6071a8af4/b: creating writer at 1733154387697 (+13 ms)Flushing 3b283966103e3011f6c81bd6071a8af4/b: appending metadata at 1733154387713 (+16 ms)Flushing 3b283966103e3011f6c81bd6071a8af4/b: closing flushed file at 1733154387713Flushing 3b283966103e3011f6c81bd6071a8af4/c: creating writer at 1733154388136 (+423 ms)Flushing 3b283966103e3011f6c81bd6071a8af4/c: appending metadata at 1733154388157 (+21 ms)Flushing 3b283966103e3011f6c81bd6071a8af4/c: closing flushed file at 1733154388157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70fcac9c: reopening flushed file at 1733154388174 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59c9e2e5: reopening flushed file at 1733154388183 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a3cbc89: reopening flushed file at 1733154388190 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 3b283966103e3011f6c81bd6071a8af4 in 537ms, sequenceid=6000, compaction requested=false; wal=null at 1733154388198 (+8 ms)Cleaning up temporary data from old regions at 1733154388200 (+2 ms)Region opened successfully at 1733154388210 (+10 ms) 2024-12-02T15:46:28,211 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-02T15:46:28,212 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3b283966103e3011f6c81bd6071a8af4, disabling compactions & flushes 2024-12-02T15:46:28,212 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:28,212 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:28,212 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. after waiting 0 ms 2024-12-02T15:46:28,212 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:28,214 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733154386688.3b283966103e3011f6c81bd6071a8af4. 2024-12-02T15:46:28,214 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3b283966103e3011f6c81bd6071a8af4: Waiting for close lock at 1733154388211Disabling compacts and flushes for region at 1733154388212 (+1 ms)Disabling writes for close at 1733154388212Writing region close event to WAL at 1733154388214 (+2 ms)Closed at 1733154388214 2024-12-02T15:46:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741870_1047 (size=175) 2024-12-02T15:46:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741870_1047 (size=175) 2024-12-02T15:46:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741870_1047 (size=175) 2024-12-02T15:46:28,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:28,222 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733154387475) 2024-12-02T15:46:28,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402 (was 395) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:55234 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33218 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:48068 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33226 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 401), ProcessCount=11 (was 11), AvailableMemoryMB=9819 (was 10054) 2024-12-02T15:46:28,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=9819 2024-12-02T15:46:28,261 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:28,267 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:28,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733154388268 2024-12-02T15:46:28,298 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268 2024-12-02T15:46:28,303 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:28,305 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ddae480de1dae84c4944ad75dca7022, NAME => 'testSequentialEditLogSeqNum,,1733154388262.0ddae480de1dae84c4944ad75dca7022.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:28,305 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733154388262.0ddae480de1dae84c4944ad75dca7022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:28,305 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,305 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,306 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022 doesn't exist for region: 0ddae480de1dae84c4944ad75dca7022 on table testSequentialEditLogSeqNum 2024-12-02T15:46:28,307 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 0ddae480de1dae84c4944ad75dca7022 on table testSequentialEditLogSeqNum 2024-12-02T15:46:28,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741875_1052 (size=62) 2024-12-02T15:46:28,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741875_1052 (size=62) 2024-12-02T15:46:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741875_1052 (size=62) 2024-12-02T15:46:28,316 INFO [StoreOpener-0ddae480de1dae84c4944ad75dca7022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,317 INFO [StoreOpener-0ddae480de1dae84c4944ad75dca7022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ddae480de1dae84c4944ad75dca7022 columnFamilyName a 2024-12-02T15:46:28,317 DEBUG [StoreOpener-0ddae480de1dae84c4944ad75dca7022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,318 INFO [StoreOpener-0ddae480de1dae84c4944ad75dca7022-1 {}] regionserver.HStore(327): Store=0ddae480de1dae84c4944ad75dca7022/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,318 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,319 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,320 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,322 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0ddae480de1dae84c4944ad75dca7022 2024-12-02T15:46:28,324 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:28,324 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0ddae480de1dae84c4944ad75dca7022; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73035942, jitterRate=0.08832034468650818}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:28,325 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0ddae480de1dae84c4944ad75dca7022: Writing region info on filesystem at 1733154388305Initializing all the Stores at 1733154388315 (+10 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388315Cleaning up temporary data from old regions at 1733154388320 (+5 ms)Region opened successfully at 1733154388325 (+5 ms) 2024-12-02T15:46:28,338 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0ddae480de1dae84c4944ad75dca7022 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-02T15:46:28,361 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/.tmp/a/10edbc159c7f431db6642216e69c8195 is 81, key is testSequentialEditLogSeqNum/a:x0/1733154388325/Put/seqid=0 2024-12-02T15:46:28,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741876_1053 (size=5833) 2024-12-02T15:46:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741876_1053 (size=5833) 2024-12-02T15:46:28,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741876_1053 (size=5833) 2024-12-02T15:46:28,371 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/.tmp/a/10edbc159c7f431db6642216e69c8195 2024-12-02T15:46:28,378 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/.tmp/a/10edbc159c7f431db6642216e69c8195 as hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/a/10edbc159c7f431db6642216e69c8195 2024-12-02T15:46:28,386 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/a/10edbc159c7f431db6642216e69c8195, entries=10, sequenceid=13, filesize=5.7 K 2024-12-02T15:46:28,388 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 0ddae480de1dae84c4944ad75dca7022 in 50ms, sequenceid=13, compaction requested=false 2024-12-02T15:46:28,388 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0ddae480de1dae84c4944ad75dca7022: 2024-12-02T15:46:28,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T15:46:28,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T15:46:28,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T15:46:28,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T15:46:28,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T15:46:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741874_1051 (size=3256) 2024-12-02T15:46:28,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741874_1051 (size=3256) 2024-12-02T15:46:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741874_1051 (size=3256) 2024-12-02T15:46:28,410 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268, size=3.2 K (3256bytes) 2024-12-02T15:46:28,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268 2024-12-02T15:46:28,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268 after 0ms 2024-12-02T15:46:28,436 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:28,436 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268 took 26ms 2024-12-02T15:46:28,439 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268 so closing down 2024-12-02T15:46:28,439 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:28,440 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733154388268.temp 2024-12-02T15:46:28,465 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000003-wal.1733154388268.temp 2024-12-02T15:46:28,466 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741877_1054 (size=2650) 2024-12-02T15:46:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741877_1054 (size=2650) 2024-12-02T15:46:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741877_1054 (size=2650) 2024-12-02T15:46:28,477 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000003-wal.1733154388268.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:28,479 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000003-wal.1733154388268.temp to hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000020 2024-12-02T15:46:28,479 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-02T15:46:28,479 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268, journal: Splitting hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268, size=3.2 K (3256bytes) at 1733154388411Finishing writing output for hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268 so closing down at 1733154388439 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000003-wal.1733154388268.temp at 1733154388465 (+26 ms)3 split writer threads finished at 1733154388466 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000003-wal.1733154388268.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733154388477 (+11 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000003-wal.1733154388268.temp to hdfs://localhost:33469/hbase/data/default/testSequentialEditLogSeqNum/0ddae480de1dae84c4944ad75dca7022/recovered.edits/0000000000000000020 at 1733154388479 (+2 ms)Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:33469/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733154388260/wal.1733154388268, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733154388479 2024-12-02T15:46:28,492 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:55234 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33218 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:33226 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 401), ProcessCount=11 (was 11), AvailableMemoryMB=9816 (was 9819) 2024-12-02T15:46:28,504 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=9816 2024-12-02T15:46:28,517 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:28,520 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:28,539 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:28,542 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72844652, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-72844652, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:28,554 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72844652/hregion-72844652.1733154388543, exclude list is [], retry=0 2024-12-02T15:46:28,557 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:28,557 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:28,557 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:28,592 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72844652/hregion-72844652.1733154388543 2024-12-02T15:46:28,593 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:28,593 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 20b058ccbc4b8d81aa17c4c754ac915a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:28,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741879_1056 (size=70) 2024-12-02T15:46:28,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741879_1056 (size=70) 2024-12-02T15:46:28,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741879_1056 (size=70) 2024-12-02T15:46:28,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:28,603 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,604 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName a 2024-12-02T15:46:28,604 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,605 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,605 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,606 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName b 2024-12-02T15:46:28,606 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,607 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,607 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,609 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName c 2024-12-02T15:46:28,609 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,610 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,610 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,611 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,611 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,613 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,613 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,613 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:28,615 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,617 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:28,617 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20b058ccbc4b8d81aa17c4c754ac915a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70861115, jitterRate=0.05591289699077606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:28,618 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20b058ccbc4b8d81aa17c4c754ac915a: Writing region info on filesystem at 1733154388601Initializing all the Stores at 1733154388602 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388602Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388602Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388603 (+1 ms)Cleaning up temporary data from old regions at 1733154388613 (+10 ms)Region opened successfully at 1733154388618 (+5 ms) 2024-12-02T15:46:28,619 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20b058ccbc4b8d81aa17c4c754ac915a, disabling compactions & flushes 2024-12-02T15:46:28,619 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:28,619 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:28,619 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. after waiting 0 ms 2024-12-02T15:46:28,619 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:28,619 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:28,619 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20b058ccbc4b8d81aa17c4c754ac915a: Waiting for close lock at 1733154388619Disabling compacts and flushes for region at 1733154388619Disabling writes for close at 1733154388619Writing region close event to WAL at 1733154388619Closed at 1733154388619 2024-12-02T15:46:28,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741878_1055 (size=175) 2024-12-02T15:46:28,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741878_1055 (size=175) 2024-12-02T15:46:28,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741878_1055 (size=175) 2024-12-02T15:46:28,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:28,625 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72844652:(num 1733154388543) 2024-12-02T15:46:28,625 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:28,627 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:28,639 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628, exclude list is [], retry=0 2024-12-02T15:46:28,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:28,643 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:28,643 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:28,666 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 2024-12-02T15:46:28,666 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:28,666 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20b058ccbc4b8d81aa17c4c754ac915a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:28,667 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:28,667 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,667 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,668 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,670 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName a 2024-12-02T15:46:28,670 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,670 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,671 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,671 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName b 2024-12-02T15:46:28,671 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,672 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,672 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,673 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName c 2024-12-02T15:46:28,673 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:28,673 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:28,674 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,675 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,676 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:28,677 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:28,678 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20b058ccbc4b8d81aa17c4c754ac915a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64179611, jitterRate=-0.04364927113056183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:28,679 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20b058ccbc4b8d81aa17c4c754ac915a: Writing region info on filesystem at 1733154388667Initializing all the Stores at 1733154388668 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388668Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388668Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154388668Cleaning up temporary data from old regions at 1733154388676 (+8 ms)Region opened successfully at 1733154388679 (+3 ms) 2024-12-02T15:46:28,682 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733154388681/Put/seqid=0 2024-12-02T15:46:28,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741881_1058 (size=4826) 2024-12-02T15:46:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741881_1058 (size=4826) 2024-12-02T15:46:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741881_1058 (size=4826) 2024-12-02T15:46:28,690 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33469/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 20b058ccbc4b8d81aa17c4c754ac915a/a 2024-12-02T15:46:28,700 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-02T15:46:28,700 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T15:46:28,700 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 20b058ccbc4b8d81aa17c4c754ac915a: 2024-12-02T15:46:28,702 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_ 2024-12-02T15:46:28,703 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33469/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 20b058ccbc4b8d81aa17c4c754ac915a/a as hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_ - updating store file list. 2024-12-02T15:46:28,709 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:28,710 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_ into 20b058ccbc4b8d81aa17c4c754ac915a/a 2024-12-02T15:46:28,710 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33469/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 20b058ccbc4b8d81aa17c4c754ac915a/a (new location: hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_) 2024-12-02T15:46:28,749 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628, size=0 (0bytes) 2024-12-02T15:46:28,749 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 might be still open, length is 0 2024-12-02T15:46:28,749 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 2024-12-02T15:46:28,750 WARN [IPC Server handler 3 on default port 33469 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-02T15:46:28,750 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 after 1ms 2024-12-02T15:46:30,431 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:55142 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55142 dst: /127.0.0.1:38685 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38685 remote=/127.0.0.1:55142]. Total timeout mills is 60000, 58285 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:30,432 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:46610 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43105:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46610 dst: /127.0.0.1:43105 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:30,432 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:39806 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42367:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39806 dst: /127.0.0.1:42367 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:30,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741880_1059 (size=606) 2024-12-02T15:46:30,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741880_1059 (size=606) 2024-12-02T15:46:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741880_1059 (size=606) 2024-12-02T15:46:32,752 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 after 4003ms 2024-12-02T15:46:32,783 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:32,783 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 took 4034ms 2024-12-02T15:46:32,785 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628; continuing. 2024-12-02T15:46:32,785 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 so closing down 2024-12-02T15:46:32,785 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:32,787 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733154388628.temp 2024-12-02T15:46:32,808 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005-wal.1733154388628.temp 2024-12-02T15:46:32,809 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:32,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741882_1060 (size=329) 2024-12-02T15:46:32,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741882_1060 (size=329) 2024-12-02T15:46:32,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741882_1060 (size=329) 2024-12-02T15:46:32,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005-wal.1733154388628.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:32,816 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005-wal.1733154388628.temp to hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005 2024-12-02T15:46:32,816 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 32 ms; skipped=1; WAL=hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T15:46:32,817 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628, journal: Splitting hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628, size=0 (0bytes) at 1733154388749Finishing writing output for hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 so closing down at 1733154392785 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005-wal.1733154388628.temp at 1733154392808 (+23 ms)3 split writer threads finished at 1733154392809 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005-wal.1733154388628.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733154392815 (+6 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005-wal.1733154388628.temp to hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005 at 1733154392816 (+1 ms)Processed 2 edits across 1 Regions in 32 ms; skipped=1; WAL=hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628, size=0, length=0, corrupted=false, cancelled=false at 1733154392816 2024-12-02T15:46:32,818 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154388628 2024-12-02T15:46:32,819 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005 2024-12-02T15:46:32,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:32,821 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:32,834 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154392822, exclude list is [], retry=0 2024-12-02T15:46:32,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:32,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:32,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:32,859 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154392822 2024-12-02T15:46:32,859 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:32,860 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 20b058ccbc4b8d81aa17c4c754ac915a, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:32,860 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:32,860 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,860 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,861 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,862 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName a 2024-12-02T15:46:32,862 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:32,868 DEBUG [StoreFileOpener-20b058ccbc4b8d81aa17c4c754ac915a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:32,868 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/d44bc1e34a4a4a29be56de1aac62cc43_SeqId_3_ 2024-12-02T15:46:32,869 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:32,869 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,870 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName b 2024-12-02T15:46:32,870 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:32,871 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:32,871 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,872 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20b058ccbc4b8d81aa17c4c754ac915a columnFamilyName c 2024-12-02T15:46:32,872 DEBUG [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:32,872 INFO [StoreOpener-20b058ccbc4b8d81aa17c4c754ac915a-1 {}] regionserver.HStore(327): Store=20b058ccbc4b8d81aa17c4c754ac915a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:32,872 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,875 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,875 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005 2024-12-02T15:46:32,898 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:32,899 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005 2024-12-02T15:46:32,899 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 20b058ccbc4b8d81aa17c4c754ac915a 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-02T15:46:32,911 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/.tmp/a/2a5fac2b2cda478a97874821f99070d8 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733154388715/Put/seqid=0 2024-12-02T15:46:32,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741884_1062 (size=5149) 2024-12-02T15:46:32,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741884_1062 (size=5149) 2024-12-02T15:46:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741884_1062 (size=5149) 2024-12-02T15:46:32,917 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/.tmp/a/2a5fac2b2cda478a97874821f99070d8 2024-12-02T15:46:32,922 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/.tmp/a/2a5fac2b2cda478a97874821f99070d8 as hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/2a5fac2b2cda478a97874821f99070d8 2024-12-02T15:46:32,928 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/a/2a5fac2b2cda478a97874821f99070d8, entries=1, sequenceid=5, filesize=5.0 K 2024-12-02T15:46:32,928 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 20b058ccbc4b8d81aa17c4c754ac915a in 29ms, sequenceid=5, compaction requested=false; wal=null 2024-12-02T15:46:32,929 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/0000000000000000005 2024-12-02T15:46:32,930 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,930 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,931 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:32,932 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 20b058ccbc4b8d81aa17c4c754ac915a 2024-12-02T15:46:32,935 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/20b058ccbc4b8d81aa17c4c754ac915a/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-02T15:46:32,936 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 20b058ccbc4b8d81aa17c4c754ac915a; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71665955, jitterRate=0.06790594756603241}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:32,936 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 20b058ccbc4b8d81aa17c4c754ac915a: Writing region info on filesystem at 1733154392860Initializing all the Stores at 1733154392861 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154392861Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154392861Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154392861Obtaining lock to block concurrent updates at 1733154392899 (+38 ms)Preparing flush snapshotting stores in 20b058ccbc4b8d81aa17c4c754ac915a at 1733154392899Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733154392899Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. at 1733154392899Flushing 20b058ccbc4b8d81aa17c4c754ac915a/a: creating writer at 1733154392899Flushing 20b058ccbc4b8d81aa17c4c754ac915a/a: appending metadata at 1733154392910 (+11 ms)Flushing 20b058ccbc4b8d81aa17c4c754ac915a/a: closing flushed file at 1733154392910Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d618a6c: reopening flushed file at 1733154392921 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 20b058ccbc4b8d81aa17c4c754ac915a in 29ms, sequenceid=5, compaction requested=false; wal=null at 1733154392928 (+7 ms)Cleaning up temporary data from old regions at 1733154392930 (+2 ms)Region opened successfully at 1733154392936 (+6 ms) 2024-12-02T15:46:32,940 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 20b058ccbc4b8d81aa17c4c754ac915a, disabling compactions & flushes 2024-12-02T15:46:32,940 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:32,940 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:32,940 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. after waiting 0 ms 2024-12-02T15:46:32,940 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:32,941 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733154388518.20b058ccbc4b8d81aa17c4c754ac915a. 2024-12-02T15:46:32,941 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 20b058ccbc4b8d81aa17c4c754ac915a: Waiting for close lock at 1733154392940Disabling compacts and flushes for region at 1733154392940Disabling writes for close at 1733154392940Writing region close event to WAL at 1733154392940Closed at 1733154392940 2024-12-02T15:46:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741883_1061 (size=175) 2024-12-02T15:46:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741883_1061 (size=175) 2024-12-02T15:46:32,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741883_1061 (size=175) 2024-12-02T15:46:32,945 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:32,945 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733154392822) 2024-12-02T15:46:32,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 407) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33469 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1132970651_22 at /127.0.0.1:39810 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33469 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1132970651_22 at /127.0.0.1:46638 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=377 (was 401), ProcessCount=11 (was 11), AvailableMemoryMB=9820 (was 9816) - AvailableMemoryMB LEAK? - 2024-12-02T15:46:32,970 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=377, ProcessCount=11, AvailableMemoryMB=9819 2024-12-02T15:46:32,986 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:32,991 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T15:46:32,994 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:32,996 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1690102a 2024-12-02T15:46:32,997 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T15:46:32,999 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T15:46:33,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T15:46:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-02T15:46:33,010 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T15:46:33,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-02T15:46:33,012 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:33,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T15:46:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T15:46:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741885_1063 (size=694) 2024-12-02T15:46:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741885_1063 (size=694) 2024-12-02T15:46:33,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741885_1063 (size=694) 2024-12-02T15:46:33,026 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82 2024-12-02T15:46:33,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741886_1064 (size=77) 2024-12-02T15:46:33,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741886_1064 (size=77) 2024-12-02T15:46:33,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741886_1064 (size=77) 2024-12-02T15:46:33,038 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:33,039 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:33,039 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,039 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,039 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:33,039 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,039 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,039 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154393039Disabling compacts and flushes for region at 1733154393039Disabling writes for close at 1733154393039Writing region close event to WAL at 1733154393039Closed at 1733154393039 2024-12-02T15:46:33,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T15:46:33,044 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733154393040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733154393040"}]},"ts":"1733154393040"} 2024-12-02T15:46:33,047 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T15:46:33,049 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T15:46:33,051 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733154393049"}]},"ts":"1733154393049"} 2024-12-02T15:46:33,054 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-02T15:46:33,055 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e1c0b6c2739c=0} racks are {/default-rack=0} 2024-12-02T15:46:33,056 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T15:46:33,057 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T15:46:33,057 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T15:46:33,057 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T15:46:33,057 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T15:46:33,057 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T15:46:33,057 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T15:46:33,057 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T15:46:33,057 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T15:46:33,057 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T15:46:33,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN}] 2024-12-02T15:46:33,061 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN 2024-12-02T15:46:33,062 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN; state=OFFLINE, location=e1c0b6c2739c,44999,1733154377173; forceNewPlan=false, retain=false 2024-12-02T15:46:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T15:46:33,214 INFO [e1c0b6c2739c:42629 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T15:46:33,215 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPENING, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:33,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN because future has completed 2024-12-02T15:46:33,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173}] 2024-12-02T15:46:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T15:46:33,385 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,385 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:33,386 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,386 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:33,386 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,386 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,388 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,389 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf1 2024-12-02T15:46:33,390 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:33,390 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:33,390 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,392 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf2 2024-12-02T15:46:33,392 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:33,393 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:33,393 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,394 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:33,394 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:33,395 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,395 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,396 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T15:46:33,397 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,399 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:33,400 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 86be1823288224745d6875510237d50b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71713981, jitterRate=0.06862159073352814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T15:46:33,400 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,400 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 86be1823288224745d6875510237d50b: Running coprocessor pre-open hook at 1733154393386Writing region info on filesystem at 1733154393386Initializing all the Stores at 1733154393387 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154393387Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154393387Cleaning up temporary data from old regions at 1733154393395 (+8 ms)Running coprocessor post-open hooks at 1733154393400 (+5 ms)Region opened successfully at 1733154393400 2024-12-02T15:46:33,402 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., pid=6, masterSystemTime=1733154393373 2024-12-02T15:46:33,405 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,405 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPEN, openSeqNum=2, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:33,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173 because future has completed 2024-12-02T15:46:33,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T15:46:33,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173 in 192 msec 2024-12-02T15:46:33,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T15:46:33,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN in 357 msec 2024-12-02T15:46:33,419 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T15:46:33,419 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733154393419"}]},"ts":"1733154393419"} 2024-12-02T15:46:33,422 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-02T15:46:33,423 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T15:46:33,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 420 msec 2024-12-02T15:46:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T15:46:33,653 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-02T15:46:33,653 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-02T15:46:33,656 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T15:46:33,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-02T15:46:33,663 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T15:46:33,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-02T15:46:33,678 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=2] 2024-12-02T15:46:33,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=86be1823288224745d6875510237d50b, source=e1c0b6c2739c,44999,1733154377173, destination=e1c0b6c2739c,40775,1733154377022, warming up region on e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T15:46:33,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=86be1823288224745d6875510237d50b, source=e1c0b6c2739c,44999,1733154377173, destination=e1c0b6c2739c,40775,1733154377022, running balancer 2024-12-02T15:46:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE 2024-12-02T15:46:33,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45487, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T15:46:33,699 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE 2024-12-02T15:46:33,701 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=CLOSING, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:33,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(7855): Warmup {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:33,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE because future has completed 2024-12-02T15:46:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:33,705 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T15:46:33,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173}] 2024-12-02T15:46:33,705 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,707 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf1 2024-12-02T15:46:33,707 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:33,707 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:33,707 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,708 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf2 2024-12-02T15:46:33,708 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:33,709 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:33,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154393709Disabling compacts and flushes for region at 1733154393709Disabling writes for close at 1733154393709Writing region close event to WAL at 1733154393710 (+1 ms)Closed at 1733154393710 2024-12-02T15:46:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-02T15:46:33,867 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,867 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T15:46:33,867 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:33,868 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,868 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,868 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:33,868 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,868 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 86be1823288224745d6875510237d50b 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-02T15:46:33,886 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/efa495b512494e888f3f41600cc65d04 is 35, key is r1/cf1:q/1733154393681/Put/seqid=0 2024-12-02T15:46:33,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741887_1065 (size=4783) 2024-12-02T15:46:33,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741887_1065 (size=4783) 2024-12-02T15:46:33,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741887_1065 (size=4783) 2024-12-02T15:46:33,892 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/efa495b512494e888f3f41600cc65d04 2024-12-02T15:46:33,900 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/efa495b512494e888f3f41600cc65d04 as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04 2024-12-02T15:46:33,907 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04, entries=1, sequenceid=5, filesize=4.7 K 2024-12-02T15:46:33,908 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 86be1823288224745d6875510237d50b in 40ms, sequenceid=5, compaction requested=false 2024-12-02T15:46:33,908 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-02T15:46:33,913 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-02T15:46:33,915 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:33,916 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154393867Running coprocessor pre-close hooks at 1733154393867Disabling compacts and flushes for region at 1733154393867Disabling writes for close at 1733154393868 (+1 ms)Obtaining lock to block concurrent updates at 1733154393868Preparing flush snapshotting stores in 86be1823288224745d6875510237d50b at 1733154393868Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733154393868Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. at 1733154393869 (+1 ms)Flushing 86be1823288224745d6875510237d50b/cf1: creating writer at 1733154393869Flushing 86be1823288224745d6875510237d50b/cf1: appending metadata at 1733154393886 (+17 ms)Flushing 86be1823288224745d6875510237d50b/cf1: closing flushed file at 1733154393886Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56a9d94e: reopening flushed file at 1733154393899 (+13 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 86be1823288224745d6875510237d50b in 40ms, sequenceid=5, compaction requested=false at 1733154393908 (+9 ms)Writing region close event to WAL at 1733154393909 (+1 ms)Running coprocessor post-close hooks at 1733154393913 (+4 ms)Closed at 1733154393915 (+2 ms) 2024-12-02T15:46:33,916 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 86be1823288224745d6875510237d50b move to e1c0b6c2739c,40775,1733154377022 record at close sequenceid=5 2024-12-02T15:46:33,919 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 86be1823288224745d6875510237d50b 2024-12-02T15:46:33,919 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=CLOSED 2024-12-02T15:46:33,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173 because future has completed 2024-12-02T15:46:33,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T15:46:33,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173 in 218 msec 2024-12-02T15:46:33,927 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE; state=CLOSED, location=e1c0b6c2739c,40775,1733154377022; forceNewPlan=false, retain=false 2024-12-02T15:46:34,077 INFO [e1c0b6c2739c:42629 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T15:46:34,078 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPENING, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:34,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE because future has completed 2024-12-02T15:46:34,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022}] 2024-12-02T15:46:34,242 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,242 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:34,243 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,243 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:34,243 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,243 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,246 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,247 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf1 2024-12-02T15:46:34,247 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:34,256 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04 2024-12-02T15:46:34,256 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:34,257 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,258 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf2 2024-12-02T15:46:34,258 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:34,258 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:34,259 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,259 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:34,261 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:34,262 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,262 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,262 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T15:46:34,264 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,265 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 86be1823288224745d6875510237d50b; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65638005, jitterRate=-0.02191750705242157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T15:46:34,265 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,265 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 86be1823288224745d6875510237d50b: Running coprocessor pre-open hook at 1733154394244Writing region info on filesystem at 1733154394244Initializing all the Stores at 1733154394245 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154394245Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154394246 (+1 ms)Cleaning up temporary data from old regions at 1733154394262 (+16 ms)Running coprocessor post-open hooks at 1733154394265 (+3 ms)Region opened successfully at 1733154394265 2024-12-02T15:46:34,267 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., pid=9, masterSystemTime=1733154394234 2024-12-02T15:46:34,269 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,269 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPEN, openSeqNum=9, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:34,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022 because future has completed 2024-12-02T15:46:34,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-02T15:46:34,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022 in 193 msec 2024-12-02T15:46:34,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE in 579 msec 2024-12-02T15:46:34,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T15:46:34,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T15:46:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:57182 deadline: 1733154454312, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=40775 startCode=1733154377022. As of locationSeqNum=5. 2024-12-02T15:46:34,333 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=40775 startCode=1733154377022. As of locationSeqNum=5. 2024-12-02T15:46:34,334 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=40775 startCode=1733154377022. As of locationSeqNum=5. 2024-12-02T15:46:34,334 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,40775,1733154377022, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=40775 startCode=1733154377022. As of locationSeqNum=5. 2024-12-02T15:46:34,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T15:46:34,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T15:46:34,450 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 86be1823288224745d6875510237d50b 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-02T15:46:34,465 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/56c3fbb425274044a23de0727bd2daaf is 29, key is r1/cf1:/1733154394442/DeleteFamily/seqid=0 2024-12-02T15:46:34,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741888_1066 (size=4906) 2024-12-02T15:46:34,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741888_1066 (size=4906) 2024-12-02T15:46:34,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741888_1066 (size=4906) 2024-12-02T15:46:34,472 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,478 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,498 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf2/511ac13bc1cf41b8a7eeebda0599750d is 29, key is r1/cf2:/1733154394442/DeleteFamily/seqid=0 2024-12-02T15:46:34,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741889_1067 (size=4906) 2024-12-02T15:46:34,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741889_1067 (size=4906) 2024-12-02T15:46:34,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741889_1067 (size=4906) 2024-12-02T15:46:34,504 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf2/511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,510 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,511 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/56c3fbb425274044a23de0727bd2daaf as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,519 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,519 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf, entries=1, sequenceid=12, filesize=4.8 K 2024-12-02T15:46:34,521 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf2/511ac13bc1cf41b8a7eeebda0599750d as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,528 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,528 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d, entries=1, sequenceid=12, filesize=4.8 K 2024-12-02T15:46:34,529 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 86be1823288224745d6875510237d50b in 79ms, sequenceid=12, compaction requested=false 2024-12-02T15:46:34,529 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 86be1823288224745d6875510237d50b: 2024-12-02T15:46:34,532 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T15:46:34,534 DEBUG [Time-limited test {}] regionserver.HStore(1541): 86be1823288224745d6875510237d50b/cf1 is initiating major compaction (all files) 2024-12-02T15:46:34,534 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T15:46:34,534 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:34,535 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 86be1823288224745d6875510237d50b/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,535 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04, hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf] into tmpdir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp, totalSize=9.5 K 2024-12-02T15:46:34,537 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting efa495b512494e888f3f41600cc65d04, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733154393681 2024-12-02T15:46:34,537 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 56c3fbb425274044a23de0727bd2daaf, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-02T15:46:34,550 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 86be1823288224745d6875510237d50b#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T15:46:34,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741890_1068 (size=4626) 2024-12-02T15:46:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741890_1068 (size=4626) 2024-12-02T15:46:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741890_1068 (size=4626) 2024-12-02T15:46:34,568 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf1/40ca5d731e6e411ab3bd8efc98bd96a1 as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/40ca5d731e6e411ab3bd8efc98bd96a1 2024-12-02T15:46:34,582 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 86be1823288224745d6875510237d50b/cf1 of 86be1823288224745d6875510237d50b into 40ca5d731e6e411ab3bd8efc98bd96a1(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T15:46:34,582 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 86be1823288224745d6875510237d50b: 2024-12-02T15:46:34,582 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-02T15:46:34,582 DEBUG [Time-limited test {}] regionserver.HStore(1541): 86be1823288224745d6875510237d50b/cf2 is initiating major compaction (all files) 2024-12-02T15:46:34,582 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T15:46:34,582 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T15:46:34,582 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 86be1823288224745d6875510237d50b/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,583 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d] into tmpdir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp, totalSize=4.8 K 2024-12-02T15:46:34,583 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 511ac13bc1cf41b8a7eeebda0599750d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-02T15:46:34,590 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 86be1823288224745d6875510237d50b#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T15:46:34,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741891_1069 (size=4592) 2024-12-02T15:46:34,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741891_1069 (size=4592) 2024-12-02T15:46:34,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741891_1069 (size=4592) 2024-12-02T15:46:34,605 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/.tmp/cf2/9340817937bf4b0da0a7b717f16cb008 as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/9340817937bf4b0da0a7b717f16cb008 2024-12-02T15:46:34,613 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 86be1823288224745d6875510237d50b/cf2 of 86be1823288224745d6875510237d50b into 9340817937bf4b0da0a7b717f16cb008(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T15:46:34,613 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 86be1823288224745d6875510237d50b: 2024-12-02T15:46:34,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=86be1823288224745d6875510237d50b, source=e1c0b6c2739c,40775,1733154377022, destination=e1c0b6c2739c,44999,1733154377173, warming up region on e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:34,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=86be1823288224745d6875510237d50b, source=e1c0b6c2739c,40775,1733154377022, destination=e1c0b6c2739c,44999,1733154377173, running balancer 2024-12-02T15:46:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE 2024-12-02T15:46:34,621 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE 2024-12-02T15:46:34,623 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=CLOSING, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:34,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(7855): Warmup {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:34,624 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,626 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf1 2024-12-02T15:46:34,626 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:34,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE because future has completed 2024-12-02T15:46:34,627 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T15:46:34,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022}] 2024-12-02T15:46:34,634 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/40ca5d731e6e411ab3bd8efc98bd96a1 2024-12-02T15:46:34,640 INFO [StoreFileOpener-86be1823288224745d6875510237d50b-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,640 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,645 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04 2024-12-02T15:46:34,645 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:34,645 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,646 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf2 2024-12-02T15:46:34,646 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:34,652 INFO [StoreFileOpener-86be1823288224745d6875510237d50b-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,652 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,658 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/9340817937bf4b0da0a7b717f16cb008 2024-12-02T15:46:34,658 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:34,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44999 {}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154394658Disabling compacts and flushes for region at 1733154394658Disabling writes for close at 1733154394658Writing region close event to WAL at 1733154394660 (+2 ms)Closed at 1733154394660 2024-12-02T15:46:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-02T15:46:34,783 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,784 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T15:46:34,784 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:34,784 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,784 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,784 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:34,785 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,786 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04, hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf] to archive 2024-12-02T15:46:34,793 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T15:46:34,797 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04 to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/efa495b512494e888f3f41600cc65d04 2024-12-02T15:46:34,798 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/56c3fbb425274044a23de0727bd2daaf 2024-12-02T15:46:34,810 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d] to archive 2024-12-02T15:46:34,811 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T15:46:34,813 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/511ac13bc1cf41b8a7eeebda0599750d 2024-12-02T15:46:34,819 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-02T15:46:34,820 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:34,820 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154394784Running coprocessor pre-close hooks at 1733154394784Disabling compacts and flushes for region at 1733154394784Disabling writes for close at 1733154394785 (+1 ms)Writing region close event to WAL at 1733154394815 (+30 ms)Running coprocessor post-close hooks at 1733154394820 (+5 ms)Closed at 1733154394820 2024-12-02T15:46:34,820 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 86be1823288224745d6875510237d50b move to e1c0b6c2739c,44999,1733154377173 record at close sequenceid=12 2024-12-02T15:46:34,822 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 86be1823288224745d6875510237d50b 2024-12-02T15:46:34,823 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=CLOSED 2024-12-02T15:46:34,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022 because future has completed 2024-12-02T15:46:34,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-02T15:46:34,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022 in 200 msec 2024-12-02T15:46:34,831 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE; state=CLOSED, location=e1c0b6c2739c,44999,1733154377173; forceNewPlan=false, retain=false 2024-12-02T15:46:34,982 INFO [e1c0b6c2739c:42629 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T15:46:34,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPENING, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:34,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE because future has completed 2024-12-02T15:46:34,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173}] 2024-12-02T15:46:35,147 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,148 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:35,148 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,148 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:35,149 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,149 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,151 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,153 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf1 2024-12-02T15:46:35,153 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:35,162 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/40ca5d731e6e411ab3bd8efc98bd96a1 2024-12-02T15:46:35,162 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:35,162 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,164 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf2 2024-12-02T15:46:35,164 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:35,172 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/9340817937bf4b0da0a7b717f16cb008 2024-12-02T15:46:35,172 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:35,172 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,173 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:35,174 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:35,175 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,175 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,176 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T15:46:35,178 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,179 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 86be1823288224745d6875510237d50b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62802939, jitterRate=-0.06416328251361847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T15:46:35,179 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,180 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 86be1823288224745d6875510237d50b: Running coprocessor pre-open hook at 1733154395149Writing region info on filesystem at 1733154395149Initializing all the Stores at 1733154395150 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154395150Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154395151 (+1 ms)Cleaning up temporary data from old regions at 1733154395175 (+24 ms)Running coprocessor post-open hooks at 1733154395179 (+4 ms)Region opened successfully at 1733154395180 (+1 ms) 2024-12-02T15:46:35,181 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., pid=12, masterSystemTime=1733154395140 2024-12-02T15:46:35,183 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,183 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,184 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPEN, openSeqNum=18, regionLocation=e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:35,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173 because future has completed 2024-12-02T15:46:35,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-02T15:46:35,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,44999,1733154377173 in 201 msec 2024-12-02T15:46:35,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, REOPEN/MOVE in 571 msec 2024-12-02T15:46:35,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T15:46:35,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T15:46:35,226 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server e1c0b6c2739c,44999,1733154377173: testing ***** 2024-12-02T15:46:35,226 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-02T15:46:35,228 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-02T15:46:35,229 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-02T15:46:35,235 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-02T15:46:35,237 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-02T15:46:35,249 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 478530568 }, "NonHeapMemoryUsage": { "committed": 171573248, "init": 7667712, "max": -1, "used": 169048208 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e1c0b6c2739c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 41, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 41, "ProcessCallTime_99th_percentile": 41, "ProcessCallTime_99.9th_percentile": 41, "ProcessCallTime_TimeRangeCount_0-1": 13, "ProcessCallTime_TimeRangeCount_1-3": 11, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 41, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 41, "TotalCallTime_99th_percentile": 41, "TotalCallTime_99.9th_percentile": 41, "TotalCallTime_TimeRangeCount_0-1": 13, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5781 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e1c0b6c2739c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53141", "tag.serverName": "e1c0b6c2739c,39549,1733154377118", "tag.clusterId": "118ff452-4204-444e-bcdf-b6b51bcf4b4f", "tag.Context": "regionserver", "tag.Hostname": "e1c0b6c2739c", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733154377118, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-02T15:46:35,252 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42629 {}] master.MasterRpcServices(700): e1c0b6c2739c,44999,1733154377173 reported a fatal error: ***** ABORTING region server e1c0b6c2739c,44999,1733154377173: testing ***** 2024-12-02T15:46:35,254 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1c0b6c2739c,44999,1733154377173' ***** 2024-12-02T15:46:35,254 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-02T15:46:35,255 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T15:46:35,255 INFO [RS:2;e1c0b6c2739c:44999 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-02T15:46:35,255 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T15:46:35,255 INFO [RS:2;e1c0b6c2739c:44999 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-02T15:46:35,255 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(3091): Received CLOSE for 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,256 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(956): aborting server e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:35,256 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T15:46:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40775 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:46074 deadline: 1733154455256, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=44999 startCode=1733154377173. As of locationSeqNum=12. 2024-12-02T15:46:35,256 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:35,256 INFO [RS:2;e1c0b6c2739c:44999 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e1c0b6c2739c:44999. 2024-12-02T15:46:35,256 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,256 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,256 DEBUG [RS:2;e1c0b6c2739c:44999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T15:46:35,256 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:35,256 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,256 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,40775,1733154377022, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,40775,1733154377022, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=44999 startCode=1733154377173. As of locationSeqNum=12. 2024-12-02T15:46:35,256 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,40775,1733154377022, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=44999 startCode=1733154377173. As of locationSeqNum=12. 2024-12-02T15:46:35,256 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,40775,1733154377022, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e1c0b6c2739c port=44999 startCode=1733154377173. As of locationSeqNum=12. 2024-12-02T15:46:35,257 DEBUG [RS:2;e1c0b6c2739c:44999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:35,257 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T15:46:35,257 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T15:46:35,257 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T15:46:35,257 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T15:46:35,258 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T15:46:35,258 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T15:46:35,258 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 86be1823288224745d6875510237d50b=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.} 2024-12-02T15:46:35,258 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T15:46:35,258 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T15:46:35,258 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T15:46:35,258 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T15:46:35,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e1c0b6c2739c,44999,1733154377173 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:35,258 DEBUG [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 86be1823288224745d6875510237d50b 2024-12-02T15:46:35,258 INFO [regionserver/e1c0b6c2739c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T15:46:35,258 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,259 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154395256Running coprocessor pre-close hooks at 1733154395256Disabling compacts and flushes for region at 1733154395256Disabling writes for close at 1733154395256Writing region close event to WAL at 1733154395258 (+2 ms)Running coprocessor post-close hooks at 1733154395258Closed at 1733154395258 2024-12-02T15:46:35,259 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:35,259 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e1c0b6c2739c,44999,1733154377173 aborting 2024-12-02T15:46:35,260 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e1c0b6c2739c,44999,1733154377173 aborting 2024-12-02T15:46:35,260 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 from cache 2024-12-02T15:46:35,264 ERROR [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-02T15:46:35,264 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T15:46:35,264 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T15:46:35,264 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733154395258Running coprocessor pre-close hooks at 1733154395258Disabling compacts and flushes for region at 1733154395258Disabling writes for close at 1733154395258Writing region close event to WAL at 1733154395264 (+6 ms)Running coprocessor post-close hooks at 1733154395264Closed at 1733154395264 2024-12-02T15:46:35,265 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T15:46:35,369 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:35,370 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1] 2024-12-02T15:46:35,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e1c0b6c2739c,44999,1733154377173 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:35,372 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e1c0b6c2739c,44999,1733154377173 aborting 2024-12-02T15:46:35,372 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e1c0b6c2739c,44999,1733154377173 aborting 2024-12-02T15:46:35,372 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 from cache 2024-12-02T15:46:35,459 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(976): stopping server e1c0b6c2739c,44999,1733154377173; all regions closed. 2024-12-02T15:46:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741836_1012 (size=7323) 2024-12-02T15:46:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741836_1012 (size=7323) 2024-12-02T15:46:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741836_1012 (size=7323) 2024-12-02T15:46:35,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741834_1010 (size=2288) 2024-12-02T15:46:35,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741834_1010 (size=2288) 2024-12-02T15:46:35,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741834_1010 (size=2288) 2024-12-02T15:46:35,466 DEBUG [RS:2;e1c0b6c2739c:44999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:35,466 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T15:46:35,466 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T15:46:35,466 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.ChoreService(370): Chore service for: regionserver/e1c0b6c2739c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T15:46:35,466 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T15:46:35,466 INFO [regionserver/e1c0b6c2739c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T15:46:35,467 INFO [RS:2;e1c0b6c2739c:44999 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44999 2024-12-02T15:46:35,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T15:46:35,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1c0b6c2739c,44999,1733154377173 2024-12-02T15:46:35,518 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T15:46:35,520 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1c0b6c2739c,44999,1733154377173] 2024-12-02T15:46:35,539 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1c0b6c2739c,44999,1733154377173 already deleted, retry=false 2024-12-02T15:46:35,539 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e1c0b6c2739c,44999,1733154377173 on e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:35,545 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e1c0b6c2739c,44999,1733154377173, splitWal=true, meta=true 2024-12-02T15:46:35,548 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for e1c0b6c2739c,44999,1733154377173 (carryingMeta=true) e1c0b6c2739c,44999,1733154377173/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7ce21979[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-02T15:46:35,548 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e1c0b6c2739c,44999,1733154377173, splitWal=true, meta=true 2024-12-02T15:46:35,550 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure e1c0b6c2739c,44999,1733154377173, splitWal=true, meta=true, isMeta: true 2024-12-02T15:46:35,551 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting 2024-12-02T15:46:35,552 INFO [PEWorker-1 {}] master.SplitWALManager(105): e1c0b6c2739c,44999,1733154377173 WAL count=1, meta=true 2024-12-02T15:46:35,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta}] 2024-12-02T15:46:35,559 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:35,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, worker=e1c0b6c2739c,39549,1733154377118}] 2024-12-02T15:46:35,579 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:35,581 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1] 2024-12-02T15:46:35,582 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e1c0b6c2739c:44999 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e1c0b6c2739c/172.17.0.3:44999 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:35,583 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1, error=java.net.ConnectException: Call to address=e1c0b6c2739c:44999 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e1c0b6c2739c/172.17.0.3:44999 2024-12-02T15:46:35,583 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-02T15:46:35,583 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 from cache 2024-12-02T15:46:35,584 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address e1c0b6c2739c:44999 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e1c0b6c2739c/172.17.0.3:44999 2024-12-02T15:46:35,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:35,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44999-0x1019802a9bc0003, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:35,632 INFO [RS:2;e1c0b6c2739c:44999 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T15:46:35,632 INFO [RS:2;e1c0b6c2739c:44999 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1c0b6c2739c,44999,1733154377173; zookeeper connection closed. 2024-12-02T15:46:35,632 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39f062b6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39f062b6 2024-12-02T15:46:35,719 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T15:46:35,721 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T15:46:35,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-02T15:46:35,738 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, size=7.2 K (7323bytes) 2024-12-02T15:46:35,738 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta 2024-12-02T15:46:35,739 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta after 1ms 2024-12-02T15:46:35,767 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:35,767 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta took 29ms 2024-12-02T15:46:35,775 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-02T15:46:35,776 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta so closing down 2024-12-02T15:46:35,776 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:35,777 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp 2024-12-02T15:46:35,802 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp 2024-12-02T15:46:35,803 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:35,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741892_1070 (size=7071) 2024-12-02T15:46:35,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741892_1070 (size=7071) 2024-12-02T15:46:35,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741892_1070 (size=7071) 2024-12-02T15:46:35,813 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:35,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T15:46:35,815 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 48 ms; skipped=1; WAL=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-02T15:46:35,816 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, journal: Splitting hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, size=7.2 K (7323bytes) at 1733154395738Finishing writing output for hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta so closing down at 1733154395776 (+38 ms)Creating recovered edits writer path=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp at 1733154395802 (+26 ms)3 split writer threads finished at 1733154395803 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733154395813 (+10 ms)Rename recovered edits hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta.temp to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733154395815 (+2 ms)Processed 16 edits across 1 Regions in 48 ms; skipped=1; WAL=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733154395816 (+1 ms) 2024-12-02T15:46:35,816 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta 2024-12-02T15:46:35,818 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-02T15:46:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-02T15:46:35,826 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs 2024-12-02T15:46:35,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-02T15:46:35,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, worker=e1c0b6c2739c,39549,1733154377118 in 265 msec 2024-12-02T15:46:35,831 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:35,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T15:46:35,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure e1c0b6c2739c%2C44999%2C1733154377173.meta.1733154379212.meta, worker=e1c0b6c2739c,39549,1733154377118 in 278 msec 2024-12-02T15:46:35,836 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting dir is empty, no logs to split. 2024-12-02T15:46:35,837 INFO [PEWorker-1 {}] master.SplitWALManager(105): e1c0b6c2739c,44999,1733154377173 WAL count=0, meta=true 2024-12-02T15:46:35,837 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e1c0b6c2739c,44999,1733154377173 WAL splitting is done? wals=0, meta=true 2024-12-02T15:46:35,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T15:46:35,841 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T15:46:35,842 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T15:46:35,889 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:35,891 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1] 2024-12-02T15:46:35,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to e1c0b6c2739c:44999 this server is in the failed servers list 2024-12-02T15:46:35,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e1c0b6c2739c:44999 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:35,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:35,892 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 from cache 2024-12-02T15:46:35,992 DEBUG [e1c0b6c2739c:42629 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-02T15:46:35,993 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(204): Hosts are {e1c0b6c2739c=0} racks are {/default-rack=0} 2024-12-02T15:46:35,993 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T15:46:35,993 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T15:46:35,993 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T15:46:35,993 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T15:46:35,993 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T15:46:35,993 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T15:46:35,993 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T15:46:35,994 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:35,995 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1c0b6c2739c,40775,1733154377022, state=OPENING 2024-12-02T15:46:36,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:36,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:36,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:36,035 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T15:46:36,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:36,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e1c0b6c2739c,40775,1733154377022}] 2024-12-02T15:46:36,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:36,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:36,196 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T15:46:36,197 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:36,197 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T15:46:36,200 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e1c0b6c2739c%2C40775%2C1733154377022.meta, suffix=.meta, logDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022, archiveDir=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs, maxLogs=32 2024-12-02T15:46:36,215 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022/e1c0b6c2739c%2C40775%2C1733154377022.meta.1733154396200.meta, exclude list is [], retry=0 2024-12-02T15:46:36,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:36,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:36,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:36,239 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,40775,1733154377022/e1c0b6c2739c%2C40775%2C1733154377022.meta.1733154396200.meta 2024-12-02T15:46:36,239 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T15:46:36,240 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T15:46:36,240 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T15:46:36,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T15:46:36,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T15:46:36,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:36,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:36,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T15:46:36,245 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T15:46:36,245 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:36,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:36,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T15:46:36,247 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T15:46:36,247 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:36,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:36,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T15:46:36,248 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T15:46:36,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:36,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T15:46:36,249 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T15:46:36,250 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740 2024-12-02T15:46:36,251 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740 2024-12-02T15:46:36,252 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T15:46:36,277 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:36,279 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T15:46:36,279 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-02T15:46:36,296 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/info/9e8573a55cbb403684b2233e1cc801ac is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b./info:regioninfo/1733154395184/Put/seqid=0 2024-12-02T15:46:36,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741894_1072 (size=11177) 2024-12-02T15:46:36,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741894_1072 (size=11177) 2024-12-02T15:46:36,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741894_1072 (size=11177) 2024-12-02T15:46:36,304 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/info/9e8573a55cbb403684b2233e1cc801ac 2024-12-02T15:46:36,322 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/ns/1ba13b075d3440f58a90fa90115afd2a is 43, key is default/ns:d/1733154379445/Put/seqid=0 2024-12-02T15:46:36,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741895_1073 (size=5153) 2024-12-02T15:46:36,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741895_1073 (size=5153) 2024-12-02T15:46:36,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741895_1073 (size=5153) 2024-12-02T15:46:36,330 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/ns/1ba13b075d3440f58a90fa90115afd2a 2024-12-02T15:46:36,348 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/table/22d2b9c7cbcf4ecca17f3cfbcda36a9a is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733154393419/Put/seqid=0 2024-12-02T15:46:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741896_1074 (size=5431) 2024-12-02T15:46:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741896_1074 (size=5431) 2024-12-02T15:46:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741896_1074 (size=5431) 2024-12-02T15:46:36,355 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/table/22d2b9c7cbcf4ecca17f3cfbcda36a9a 2024-12-02T15:46:36,362 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/info/9e8573a55cbb403684b2233e1cc801ac as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/info/9e8573a55cbb403684b2233e1cc801ac 2024-12-02T15:46:36,370 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/info/9e8573a55cbb403684b2233e1cc801ac, entries=36, sequenceid=18, filesize=10.9 K 2024-12-02T15:46:36,371 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/ns/1ba13b075d3440f58a90fa90115afd2a as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/ns/1ba13b075d3440f58a90fa90115afd2a 2024-12-02T15:46:36,378 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/ns/1ba13b075d3440f58a90fa90115afd2a, entries=2, sequenceid=18, filesize=5.0 K 2024-12-02T15:46:36,380 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/table/22d2b9c7cbcf4ecca17f3cfbcda36a9a as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/table/22d2b9c7cbcf4ecca17f3cfbcda36a9a 2024-12-02T15:46:36,387 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/table/22d2b9c7cbcf4ecca17f3cfbcda36a9a, entries=2, sequenceid=18, filesize=5.3 K 2024-12-02T15:46:36,387 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=18, compaction requested=false; wal=null 2024-12-02T15:46:36,388 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-02T15:46:36,390 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T15:46:36,390 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T15:46:36,391 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T15:46:36,393 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T15:46:36,395 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-02T15:46:36,396 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73211933, jitterRate=0.09094281494617462}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T15:46:36,397 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T15:46:36,397 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733154396241Writing region info on filesystem at 1733154396241Initializing all the Stores at 1733154396242 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154396242Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154396242Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154396242Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733154396242Obtaining lock to block concurrent updates at 1733154396279 (+37 ms)Preparing flush snapshotting stores in 1588230740 at 1733154396279Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733154396280 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733154396280Flushing 1588230740/info: creating writer at 1733154396280Flushing 1588230740/info: appending metadata at 1733154396296 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733154396296Flushing 1588230740/ns: creating writer at 1733154396309 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733154396322 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733154396322Flushing 1588230740/table: creating writer at 1733154396335 (+13 ms)Flushing 1588230740/table: appending metadata at 1733154396348 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733154396348Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50a5e997: reopening flushed file at 1733154396361 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15bbc97d: reopening flushed file at 1733154396370 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@212d51ec: reopening flushed file at 1733154396379 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=18, compaction requested=false; wal=null at 1733154396387 (+8 ms)Cleaning up temporary data from old regions at 1733154396390 (+3 ms)Running coprocessor post-open hooks at 1733154396397 (+7 ms)Region opened successfully at 1733154396397 2024-12-02T15:46:36,398 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733154396188 2024-12-02T15:46:36,399 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:36,401 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,40775,1733154377022, seqNum=-1] 2024-12-02T15:46:36,401 DEBUG [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T15:46:36,401 INFO [RS_OPEN_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T15:46:36,402 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:36,403 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e1c0b6c2739c,40775,1733154377022, state=OPEN 2024-12-02T15:46:36,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:36,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:36,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T15:46:36,443 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:36,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:36,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:36,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T15:46:36,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-02T15:46:36,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e1c0b6c2739c,40775,1733154377022 in 408 msec 2024-12-02T15:46:36,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-02T15:46:36,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-12-02T15:46:36,449 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): e1c0b6c2739c,44999,1733154377173 had 2 regions 2024-12-02T15:46:36,451 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e1c0b6c2739c,44999,1733154377173, splitWal=true, meta=true, isMeta: false 2024-12-02T15:46:36,454 INFO [PEWorker-2 {}] master.SplitWALManager(105): e1c0b6c2739c,44999,1733154377173 WAL count=1, meta=false 2024-12-02T15:46:36,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e1c0b6c2739c%2C44999%2C1733154377173.1733154378800}] 2024-12-02T15:46:36,456 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:36,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, worker=e1c0b6c2739c,39549,1733154377118}] 2024-12-02T15:46:36,519 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to e1c0b6c2739c:44999 this server is in the failed servers list 2024-12-02T15:46:36,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e1c0b6c2739c:44999 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:36,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:36,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=12 from cache 2024-12-02T15:46:36,606 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-02T15:46:36,607 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:36,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-02T15:46:36,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:36,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39549 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-02T15:46:36,627 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, size=2.2 K (2288bytes) 2024-12-02T15:46:36,627 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 2024-12-02T15:46:36,628 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 after 1ms 2024-12-02T15:46:36,650 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:36,650 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 took 23ms 2024-12-02T15:46:36,653 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 86be1823288224745d6875510237d50b: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-02T15:46:36,654 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 so closing down 2024-12-02T15:46:36,654 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:36,654 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:36,654 INFO [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-02T15:46:36,654 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, journal: Splitting hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, size=2.2 K (2288bytes) at 1733154396627Finishing writing output for hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 so closing down at 1733154396654 (+27 ms)3 split writer threads finished at 1733154396654Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733154396654 2024-12-02T15:46:36,654 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 2024-12-02T15:46:36,654 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e1c0b6c2739c:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-02T15:46:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42629 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-02T15:46:36,658 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting/e1c0b6c2739c%2C44999%2C1733154377173.1733154378800 to hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs 2024-12-02T15:46:36,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-02T15:46:36,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, worker=e1c0b6c2739c,39549,1733154377118 in 202 msec 2024-12-02T15:46:36,663 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:36,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-02T15:46:36,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure e1c0b6c2739c%2C44999%2C1733154377173.1733154378800, worker=e1c0b6c2739c,39549,1733154377118 in 209 msec 2024-12-02T15:46:36,666 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/WALs/e1c0b6c2739c,44999,1733154377173-splitting dir is empty, no logs to split. 2024-12-02T15:46:36,666 INFO [PEWorker-2 {}] master.SplitWALManager(105): e1c0b6c2739c,44999,1733154377173 WAL count=0, meta=false 2024-12-02T15:46:36,666 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if e1c0b6c2739c,44999,1733154377173 WAL splitting is done? wals=0, meta=false 2024-12-02T15:46:36,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN}] 2024-12-02T15:46:36,670 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN 2024-12-02T15:46:36,670 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T15:46:36,735 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18] 2024-12-02T15:46:36,735 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to e1c0b6c2739c:44999 this server is in the failed servers list 2024-12-02T15:46:36,735 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e1c0b6c2739c:44999 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:36,736 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:36,736 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18 from cache 2024-12-02T15:46:36,821 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(204): Hosts are {e1c0b6c2739c=0} racks are {/default-rack=0} 2024-12-02T15:46:36,821 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T15:46:36,821 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T15:46:36,822 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T15:46:36,822 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T15:46:36,822 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T15:46:36,822 INFO [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T15:46:36,822 DEBUG [e1c0b6c2739c:42629 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T15:46:36,822 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPENING, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:36,824 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e1c0b6c2739c:44999 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e1c0b6c2739c/172.17.0.3:44999 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:36,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1, error=java.net.ConnectException: Call to address=e1c0b6c2739c:44999 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e1c0b6c2739c/172.17.0.3:44999 2024-12-02T15:46:36,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-02T15:46:36,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,44999,1733154377173, seqNum=-1 from cache 2024-12-02T15:46:36,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address e1c0b6c2739c:44999 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e1c0b6c2739c/172.17.0.3:44999 2024-12-02T15:46:36,855 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T15:46:36,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T15:46:36,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e1c0b6c2739c,40775,1733154377022, seqNum=-1] 2024-12-02T15:46:36,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T15:46:36,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49557, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T15:46:36,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN because future has completed 2024-12-02T15:46:36,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022}] 2024-12-02T15:46:37,047 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18] 2024-12-02T15:46:37,048 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to e1c0b6c2739c:44999 this server is in the failed servers list 2024-12-02T15:46:37,048 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e1c0b6c2739c:44999 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:37,049 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e1c0b6c2739c:44999 2024-12-02T15:46:37,049 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,44999,1733154377173, seqNum=18 from cache 2024-12-02T15:46:37,097 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:37,097 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 86be1823288224745d6875510237d50b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:37,097 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,097 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:37,098 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,098 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,100 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,101 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf1 2024-12-02T15:46:37,101 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,108 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf1/40ca5d731e6e411ab3bd8efc98bd96a1 2024-12-02T15:46:37,108 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,108 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,109 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86be1823288224745d6875510237d50b columnFamilyName cf2 2024-12-02T15:46:37,109 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,115 DEBUG [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/cf2/9340817937bf4b0da0a7b717f16cb008 2024-12-02T15:46:37,115 INFO [StoreOpener-86be1823288224745d6875510237d50b-1 {}] regionserver.HStore(327): Store=86be1823288224745d6875510237d50b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,115 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,116 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:37,117 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b 2024-12-02T15:46:37,118 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,118 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,118 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-02T15:46:37,120 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,120 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 86be1823288224745d6875510237d50b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62940130, jitterRate=-0.062118977308273315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-02T15:46:37,120 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86be1823288224745d6875510237d50b 2024-12-02T15:46:37,121 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 86be1823288224745d6875510237d50b: Running coprocessor pre-open hook at 1733154397098Writing region info on filesystem at 1733154397098Initializing all the Stores at 1733154397099 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397099Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397099Cleaning up temporary data from old regions at 1733154397118 (+19 ms)Running coprocessor post-open hooks at 1733154397120 (+2 ms)Region opened successfully at 1733154397121 (+1 ms) 2024-12-02T15:46:37,122 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., pid=21, masterSystemTime=1733154397093 2024-12-02T15:46:37,124 DEBUG [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:37,124 INFO [RS_OPEN_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:37,125 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=86be1823288224745d6875510237d50b, regionState=OPEN, openSeqNum=18, regionLocation=e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:37,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022 because future has completed 2024-12-02T15:46:37,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-02T15:46:37,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 86be1823288224745d6875510237d50b, server=e1c0b6c2739c,40775,1733154377022 in 190 msec 2024-12-02T15:46:37,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-02T15:46:37,133 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server e1c0b6c2739c,44999,1733154377173 after splitting done 2024-12-02T15:46:37,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=86be1823288224745d6875510237d50b, ASSIGN in 462 msec 2024-12-02T15:46:37,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure e1c0b6c2739c,44999,1733154377173, splitWal=true, meta=true in 1.5930 sec 2024-12-02T15:46:37,567 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b., hostname=e1c0b6c2739c,40775,1733154377022, seqNum=18] 2024-12-02T15:46:37,582 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 410), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 377), ProcessCount=11 (was 11), AvailableMemoryMB=9775 (was 9819) 2024-12-02T15:46:37,584 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-02T15:46:37,595 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=9775 2024-12-02T15:46:37,595 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-02T15:46:37,611 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:37,613 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:37,613 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:37,616 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33261784, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-33261784, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:37,626 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33261784/hregion-33261784.1733154397616, exclude list is [], retry=0 2024-12-02T15:46:37,628 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:37,629 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:37,629 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:37,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33261784/hregion-33261784.1733154397616 2024-12-02T15:46:37,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:37,653 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c5e6b2f4cb894c09d8c8c65e64c7b374, NAME => 'testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:37,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741898_1076 (size=67) 2024-12-02T15:46:37,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741898_1076 (size=67) 2024-12-02T15:46:37,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741898_1076 (size=67) 2024-12-02T15:46:37,662 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:37,663 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,665 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName a 2024-12-02T15:46:37,665 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,665 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,665 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,666 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName b 2024-12-02T15:46:37,667 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,667 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,667 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,668 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName c 2024-12-02T15:46:37,668 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,669 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,669 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,671 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,671 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,672 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:37,673 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,675 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:37,675 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c5e6b2f4cb894c09d8c8c65e64c7b374; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65968216, jitterRate=-0.01699697971343994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:37,676 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c5e6b2f4cb894c09d8c8c65e64c7b374: Writing region info on filesystem at 1733154397662Initializing all the Stores at 1733154397663 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397663Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397663Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397663Cleaning up temporary data from old regions at 1733154397671 (+8 ms)Region opened successfully at 1733154397676 (+5 ms) 2024-12-02T15:46:37,676 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c5e6b2f4cb894c09d8c8c65e64c7b374, disabling compactions & flushes 2024-12-02T15:46:37,676 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,676 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,676 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. after waiting 0 ms 2024-12-02T15:46:37,676 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,677 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,677 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c5e6b2f4cb894c09d8c8c65e64c7b374: Waiting for close lock at 1733154397676Disabling compacts and flushes for region at 1733154397676Disabling writes for close at 1733154397676Writing region close event to WAL at 1733154397677 (+1 ms)Closed at 1733154397677 2024-12-02T15:46:37,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741897_1075 (size=175) 2024-12-02T15:46:37,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741897_1075 (size=175) 2024-12-02T15:46:37,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741897_1075 (size=175) 2024-12-02T15:46:37,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:37,684 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33261784:(num 1733154397616) 2024-12-02T15:46:37,684 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:37,685 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:37,696 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685, exclude list is [], retry=0 2024-12-02T15:46:37,699 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:37,699 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:37,699 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:37,722 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 2024-12-02T15:46:37,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:37,722 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c5e6b2f4cb894c09d8c8c65e64c7b374, NAME => 'testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:37,722 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:37,722 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,722 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,724 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,724 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName a 2024-12-02T15:46:37,724 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,725 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,725 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,726 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName b 2024-12-02T15:46:37,726 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,726 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,726 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,727 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName c 2024-12-02T15:46:37,727 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:37,727 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:37,727 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,728 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,729 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,730 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,730 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,730 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:37,732 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:37,732 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c5e6b2f4cb894c09d8c8c65e64c7b374; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69178440, jitterRate=0.030839085578918457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:37,733 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c5e6b2f4cb894c09d8c8c65e64c7b374: Writing region info on filesystem at 1733154397722Initializing all the Stores at 1733154397723 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397723Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397723Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154397723Cleaning up temporary data from old regions at 1733154397730 (+7 ms)Region opened successfully at 1733154397733 (+3 ms) 2024-12-02T15:46:37,759 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c5e6b2f4cb894c09d8c8c65e64c7b374 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-02T15:46:37,774 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/a/5f4d6c567a80499da65f8c312d38c3a0 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733154397733/Put/seqid=0 2024-12-02T15:46:37,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741900_1078 (size=5958) 2024-12-02T15:46:37,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741900_1078 (size=5958) 2024-12-02T15:46:37,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741900_1078 (size=5958) 2024-12-02T15:46:37,781 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/a/5f4d6c567a80499da65f8c312d38c3a0 2024-12-02T15:46:37,801 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/b/3fecc931ac7c4d0c951147a1eb8e22ad is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733154397743/Put/seqid=0 2024-12-02T15:46:37,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741901_1079 (size=5958) 2024-12-02T15:46:37,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741901_1079 (size=5958) 2024-12-02T15:46:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741901_1079 (size=5958) 2024-12-02T15:46:37,808 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/b/3fecc931ac7c4d0c951147a1eb8e22ad 2024-12-02T15:46:37,834 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/c/9faa1a6fbddc426993bc2b1837d6583b is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733154397750/Put/seqid=0 2024-12-02T15:46:37,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741902_1080 (size=5958) 2024-12-02T15:46:37,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741902_1080 (size=5958) 2024-12-02T15:46:37,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741902_1080 (size=5958) 2024-12-02T15:46:37,842 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/c/9faa1a6fbddc426993bc2b1837d6583b 2024-12-02T15:46:37,847 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/a/5f4d6c567a80499da65f8c312d38c3a0 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/a/5f4d6c567a80499da65f8c312d38c3a0 2024-12-02T15:46:37,852 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/a/5f4d6c567a80499da65f8c312d38c3a0, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T15:46:37,854 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/b/3fecc931ac7c4d0c951147a1eb8e22ad as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/b/3fecc931ac7c4d0c951147a1eb8e22ad 2024-12-02T15:46:37,859 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/b/3fecc931ac7c4d0c951147a1eb8e22ad, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T15:46:37,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/c/9faa1a6fbddc426993bc2b1837d6583b as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/c/9faa1a6fbddc426993bc2b1837d6583b 2024-12-02T15:46:37,865 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/c/9faa1a6fbddc426993bc2b1837d6583b, entries=10, sequenceid=33, filesize=5.8 K 2024-12-02T15:46:37,869 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c5e6b2f4cb894c09d8c8c65e64c7b374 in 110ms, sequenceid=33, compaction requested=false 2024-12-02T15:46:37,869 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c5e6b2f4cb894c09d8c8c65e64c7b374: 2024-12-02T15:46:37,869 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c5e6b2f4cb894c09d8c8c65e64c7b374, disabling compactions & flushes 2024-12-02T15:46:37,869 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,869 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,870 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. after waiting 0 ms 2024-12-02T15:46:37,870 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,871 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. 2024-12-02T15:46:37,871 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c5e6b2f4cb894c09d8c8c65e64c7b374: Waiting for close lock at 1733154397869Disabling compacts and flushes for region at 1733154397869Disabling writes for close at 1733154397870 (+1 ms)Writing region close event to WAL at 1733154397871 (+1 ms)Closed at 1733154397871 2024-12-02T15:46:37,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741899_1077 (size=6311) 2024-12-02T15:46:37,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741899_1077 (size=6311) 2024-12-02T15:46:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741899_1077 (size=6311) 2024-12-02T15:46:37,878 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/b/3fecc931ac7c4d0c951147a1eb8e22ad to hdfs://localhost:33469/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/b/3fecc931ac7c4d0c951147a1eb8e22ad 2024-12-02T15:46:37,895 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685, size=6.2 K (6311bytes) 2024-12-02T15:46:37,895 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 2024-12-02T15:46:37,895 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 after 0ms 2024-12-02T15:46:37,921 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:37,921 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 took 27ms 2024-12-02T15:46:37,924 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 so closing down 2024-12-02T15:46:37,924 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:37,928 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733154397685.temp 2024-12-02T15:46:37,963 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000003-wal.1733154397685.temp 2024-12-02T15:46:37,964 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:37,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741903_1081 (size=5575) 2024-12-02T15:46:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741903_1081 (size=5575) 2024-12-02T15:46:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741903_1081 (size=5575) 2024-12-02T15:46:37,971 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000003-wal.1733154397685.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:37,973 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000003-wal.1733154397685.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000032 2024-12-02T15:46:37,973 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 51 ms; skipped=2; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-02T15:46:37,973 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685, journal: Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685, size=6.2 K (6311bytes) at 1733154397895Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 so closing down at 1733154397924 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000003-wal.1733154397685.temp at 1733154397963 (+39 ms)3 split writer threads finished at 1733154397964 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000003-wal.1733154397685.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733154397971 (+7 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000003-wal.1733154397685.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000032 at 1733154397973 (+2 ms)Processed 32 edits across 1 Regions in 51 ms; skipped=2; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733154397973 2024-12-02T15:46:37,975 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397685 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154397685 2024-12-02T15:46:37,976 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000032 2024-12-02T15:46:37,976 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:37,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:37,990 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397978, exclude list is [], retry=0 2024-12-02T15:46:37,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:37,994 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:37,994 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:38,018 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397978 2024-12-02T15:46:38,018 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:38,018 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c5e6b2f4cb894c09d8c8c65e64c7b374, NAME => 'testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:38,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:38,019 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,019 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,020 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,021 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName a 2024-12-02T15:46:38,021 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,026 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/a/5f4d6c567a80499da65f8c312d38c3a0 2024-12-02T15:46:38,026 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,026 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,028 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName b 2024-12-02T15:46:38,028 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,028 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,029 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,029 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5e6b2f4cb894c09d8c8c65e64c7b374 columnFamilyName c 2024-12-02T15:46:38,029 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,035 DEBUG [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/c/9faa1a6fbddc426993bc2b1837d6583b 2024-12-02T15:46:38,035 INFO [StoreOpener-c5e6b2f4cb894c09d8c8c65e64c7b374-1 {}] regionserver.HStore(327): Store=c5e6b2f4cb894c09d8c8c65e64c7b374/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,035 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,036 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,038 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000032 2024-12-02T15:46:38,062 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:38,063 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000032 2024-12-02T15:46:38,064 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c5e6b2f4cb894c09d8c8c65e64c7b374 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-02T15:46:38,077 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/b/e1e574e5cc61406f8fa5b64e232adaaf is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733154397743/Put/seqid=0 2024-12-02T15:46:38,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741905_1083 (size=5958) 2024-12-02T15:46:38,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741905_1083 (size=5958) 2024-12-02T15:46:38,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741905_1083 (size=5958) 2024-12-02T15:46:38,085 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/b/e1e574e5cc61406f8fa5b64e232adaaf 2024-12-02T15:46:38,092 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/.tmp/b/e1e574e5cc61406f8fa5b64e232adaaf as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/b/e1e574e5cc61406f8fa5b64e232adaaf 2024-12-02T15:46:38,097 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/b/e1e574e5cc61406f8fa5b64e232adaaf, entries=10, sequenceid=32, filesize=5.8 K 2024-12-02T15:46:38,097 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c5e6b2f4cb894c09d8c8c65e64c7b374 in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-12-02T15:46:38,098 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/0000000000000000032 2024-12-02T15:46:38,099 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,099 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,100 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:38,101 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c5e6b2f4cb894c09d8c8c65e64c7b374 2024-12-02T15:46:38,103 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c5e6b2f4cb894c09d8c8c65e64c7b374/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-02T15:46:38,104 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c5e6b2f4cb894c09d8c8c65e64c7b374; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72106052, jitterRate=0.07446390390396118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:38,105 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c5e6b2f4cb894c09d8c8c65e64c7b374: Writing region info on filesystem at 1733154398019Initializing all the Stores at 1733154398020 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398020Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398020Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398020Obtaining lock to block concurrent updates at 1733154398064 (+44 ms)Preparing flush snapshotting stores in c5e6b2f4cb894c09d8c8c65e64c7b374 at 1733154398064Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733154398064Flushing stores of testReplayEditsWrittenViaHRegion,,1733154397612.c5e6b2f4cb894c09d8c8c65e64c7b374. at 1733154398064Flushing c5e6b2f4cb894c09d8c8c65e64c7b374/b: creating writer at 1733154398064Flushing c5e6b2f4cb894c09d8c8c65e64c7b374/b: appending metadata at 1733154398076 (+12 ms)Flushing c5e6b2f4cb894c09d8c8c65e64c7b374/b: closing flushed file at 1733154398077 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dd1cbb9: reopening flushed file at 1733154398091 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c5e6b2f4cb894c09d8c8c65e64c7b374 in 33ms, sequenceid=32, compaction requested=false; wal=null at 1733154398097 (+6 ms)Cleaning up temporary data from old regions at 1733154398099 (+2 ms)Region opened successfully at 1733154398105 (+6 ms) 2024-12-02T15:46:38,139 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:47636 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:58380 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:49450 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:55210 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:39810 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:46638 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=9750 (was 9775) 2024-12-02T15:46:38,139 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-02T15:46:38,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=9750 2024-12-02T15:46:38,156 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-02T15:46:38,172 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:38,174 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:38,174 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:38,176 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99719029, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-99719029, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:38,188 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99719029/hregion-99719029.1733154398177, exclude list is [], retry=0 2024-12-02T15:46:38,191 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:38,191 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:38,191 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:38,217 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99719029/hregion-99719029.1733154398177 2024-12-02T15:46:38,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:38,218 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2d5146d261fcea0ac5ee2aef06045ae2, NAME => 'testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:38,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741907_1085 (size=68) 2024-12-02T15:46:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741907_1085 (size=68) 2024-12-02T15:46:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741907_1085 (size=68) 2024-12-02T15:46:38,229 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:38,230 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,232 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName a 2024-12-02T15:46:38,232 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,233 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,233 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,234 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName b 2024-12-02T15:46:38,234 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,235 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,235 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,236 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName c 2024-12-02T15:46:38,236 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,237 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,237 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,239 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,239 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,240 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:38,241 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,243 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:38,244 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d5146d261fcea0ac5ee2aef06045ae2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65199666, jitterRate=-0.028449267148971558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:38,244 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d5146d261fcea0ac5ee2aef06045ae2: Writing region info on filesystem at 1733154398229Initializing all the Stores at 1733154398230 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398230Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398230Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398230Cleaning up temporary data from old regions at 1733154398239 (+9 ms)Region opened successfully at 1733154398244 (+5 ms) 2024-12-02T15:46:38,245 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d5146d261fcea0ac5ee2aef06045ae2, disabling compactions & flushes 2024-12-02T15:46:38,245 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:38,245 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:38,245 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. after waiting 0 ms 2024-12-02T15:46:38,245 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:38,245 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:38,245 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d5146d261fcea0ac5ee2aef06045ae2: Waiting for close lock at 1733154398245Disabling compacts and flushes for region at 1733154398245Disabling writes for close at 1733154398245Writing region close event to WAL at 1733154398245Closed at 1733154398245 2024-12-02T15:46:38,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741906_1084 (size=175) 2024-12-02T15:46:38,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741906_1084 (size=175) 2024-12-02T15:46:38,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741906_1084 (size=175) 2024-12-02T15:46:38,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:38,250 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99719029:(num 1733154398177) 2024-12-02T15:46:38,250 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:38,251 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:38,263 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252, exclude list is [], retry=0 2024-12-02T15:46:38,266 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:38,266 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:38,267 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:38,301 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 2024-12-02T15:46:38,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:38,367 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d5146d261fcea0ac5ee2aef06045ae2, NAME => 'testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:38,370 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,370 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:38,370 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,370 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,373 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,374 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName a 2024-12-02T15:46:38,375 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,375 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,376 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,377 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName b 2024-12-02T15:46:38,377 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,377 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,378 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,379 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName c 2024-12-02T15:46:38,379 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:38,379 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:38,380 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,380 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,382 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,383 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,383 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,383 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:38,384 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,385 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d5146d261fcea0ac5ee2aef06045ae2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66425926, jitterRate=-0.01017656922340393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:38,385 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:38,385 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d5146d261fcea0ac5ee2aef06045ae2: Running coprocessor pre-open hook at 1733154398370Writing region info on filesystem at 1733154398370Initializing all the Stores at 1733154398371 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398371Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398372 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154398372Cleaning up temporary data from old regions at 1733154398383 (+11 ms)Running coprocessor post-open hooks at 1733154398385 (+2 ms)Region opened successfully at 1733154398385 2024-12-02T15:46:38,397 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d5146d261fcea0ac5ee2aef06045ae2 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-02T15:46:38,399 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:38,917 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T15:46:39,399 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:40,400 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:41,401 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:42,402 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:43,403 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:43,875 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T15:46:44,404 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:45,405 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:45,449 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T15:46:46,406 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:46,606 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-02T15:46:46,606 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:46,608 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-02T15:46:46,608 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:46,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-02T15:46:46,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:47,407 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 2d5146d261fcea0ac5ee2aef06045ae2/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:47,409 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d5146d261fcea0ac5ee2aef06045ae2: 2024-12-02T15:46:47,409 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:47,425 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d5146d261fcea0ac5ee2aef06045ae2: 2024-12-02T15:46:47,425 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-02T15:46:47,425 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d5146d261fcea0ac5ee2aef06045ae2, disabling compactions & flushes 2024-12-02T15:46:47,426 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:47,426 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:47,426 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. after waiting 0 ms 2024-12-02T15:46:47,426 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:47,426 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:47,426 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. 2024-12-02T15:46:47,426 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d5146d261fcea0ac5ee2aef06045ae2: Waiting for close lock at 1733154407425Running coprocessor pre-close hooks at 1733154407425Disabling compacts and flushes for region at 1733154407425Disabling writes for close at 1733154407426 (+1 ms)Writing region close event to WAL at 1733154407426Running coprocessor post-close hooks at 1733154407426Closed at 1733154407426 2024-12-02T15:46:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741908_1086 (size=4236) 2024-12-02T15:46:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741908_1086 (size=4236) 2024-12-02T15:46:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741908_1086 (size=4236) 2024-12-02T15:46:47,444 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252, size=4.1 K (4236bytes) 2024-12-02T15:46:47,444 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 2024-12-02T15:46:47,444 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 after 0ms 2024-12-02T15:46:47,467 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:47,467 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 took 24ms 2024-12-02T15:46:47,469 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 so closing down 2024-12-02T15:46:47,469 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:47,471 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733154398252.temp 2024-12-02T15:46:47,494 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000004-wal.1733154398252.temp 2024-12-02T15:46:47,494 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741909_1087 (size=3245) 2024-12-02T15:46:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741909_1087 (size=3245) 2024-12-02T15:46:47,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741909_1087 (size=3245) 2024-12-02T15:46:47,500 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000004-wal.1733154398252.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:47,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000004-wal.1733154398252.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000026 2024-12-02T15:46:47,502 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-02T15:46:47,502 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252, journal: Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252, size=4.1 K (4236bytes) at 1733154407444Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 so closing down at 1733154407469 (+25 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000004-wal.1733154398252.temp at 1733154407494 (+25 ms)3 split writer threads finished at 1733154407494Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000004-wal.1733154398252.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733154407500 (+6 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000004-wal.1733154398252.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000026 at 1733154407502 (+2 ms)Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733154407502 2024-12-02T15:46:47,504 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154398252 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154398252 2024-12-02T15:46:47,505 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000026 2024-12-02T15:46:47,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:47,506 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:47,518 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154407506, exclude list is [], retry=0 2024-12-02T15:46:47,520 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:47,520 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:47,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:47,546 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154407506 2024-12-02T15:46:47,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:47,547 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d5146d261fcea0ac5ee2aef06045ae2, NAME => 'testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:47,548 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,548 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:47,548 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,548 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,550 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,551 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName a 2024-12-02T15:46:47,551 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:47,551 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:47,552 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,552 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName b 2024-12-02T15:46:47,552 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:47,553 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:47,553 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,554 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d5146d261fcea0ac5ee2aef06045ae2 columnFamilyName c 2024-12-02T15:46:47,554 DEBUG [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:47,555 INFO [StoreOpener-2d5146d261fcea0ac5ee2aef06045ae2-1 {}] regionserver.HStore(327): Store=2d5146d261fcea0ac5ee2aef06045ae2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:47,555 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,557 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,557 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000026 2024-12-02T15:46:47,604 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:47,606 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000026 2024-12-02T15:46:47,606 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d5146d261fcea0ac5ee2aef06045ae2 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-02T15:46:47,620 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/a/77fdc10f39824338ab622deb4547cc04 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733154407416/Put/seqid=0 2024-12-02T15:46:47,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741911_1089 (size=5523) 2024-12-02T15:46:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741911_1089 (size=5523) 2024-12-02T15:46:47,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741911_1089 (size=5523) 2024-12-02T15:46:47,628 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/a/77fdc10f39824338ab622deb4547cc04 2024-12-02T15:46:47,652 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/b/1768e4a1c4fc48849b1530e518f07ac4 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733154407409/Put/seqid=0 2024-12-02T15:46:47,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741912_1090 (size=5524) 2024-12-02T15:46:47,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741912_1090 (size=5524) 2024-12-02T15:46:47,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741912_1090 (size=5524) 2024-12-02T15:46:47,660 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/b/1768e4a1c4fc48849b1530e518f07ac4 2024-12-02T15:46:47,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/c/f0171f7cdad64e728bea738ebe338a8f is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733154407412/Put/seqid=0 2024-12-02T15:46:47,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741913_1091 (size=5457) 2024-12-02T15:46:47,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741913_1091 (size=5457) 2024-12-02T15:46:47,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741913_1091 (size=5457) 2024-12-02T15:46:47,688 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/c/f0171f7cdad64e728bea738ebe338a8f 2024-12-02T15:46:47,694 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/a/77fdc10f39824338ab622deb4547cc04 as hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/a/77fdc10f39824338ab622deb4547cc04 2024-12-02T15:46:47,700 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/a/77fdc10f39824338ab622deb4547cc04, entries=7, sequenceid=26, filesize=5.4 K 2024-12-02T15:46:47,701 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/b/1768e4a1c4fc48849b1530e518f07ac4 as hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/b/1768e4a1c4fc48849b1530e518f07ac4 2024-12-02T15:46:47,707 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/b/1768e4a1c4fc48849b1530e518f07ac4, entries=7, sequenceid=26, filesize=5.4 K 2024-12-02T15:46:47,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/.tmp/c/f0171f7cdad64e728bea738ebe338a8f as hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/c/f0171f7cdad64e728bea738ebe338a8f 2024-12-02T15:46:47,713 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/c/f0171f7cdad64e728bea738ebe338a8f, entries=6, sequenceid=26, filesize=5.3 K 2024-12-02T15:46:47,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 2d5146d261fcea0ac5ee2aef06045ae2 in 107ms, sequenceid=26, compaction requested=false; wal=null 2024-12-02T15:46:47,714 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/0000000000000000026 2024-12-02T15:46:47,715 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,715 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,716 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:47,717 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,719 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsAfterAbortingFlush/2d5146d261fcea0ac5ee2aef06045ae2/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-02T15:46:47,720 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d5146d261fcea0ac5ee2aef06045ae2; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66182986, jitterRate=-0.01379665732383728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:47,720 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d5146d261fcea0ac5ee2aef06045ae2 2024-12-02T15:46:47,720 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d5146d261fcea0ac5ee2aef06045ae2: Running coprocessor pre-open hook at 1733154407548Writing region info on filesystem at 1733154407548Initializing all the Stores at 1733154407549 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154407549Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154407549Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154407549Obtaining lock to block concurrent updates at 1733154407606 (+57 ms)Preparing flush snapshotting stores in 2d5146d261fcea0ac5ee2aef06045ae2 at 1733154407606Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733154407606Flushing stores of testReplayEditsAfterAbortingFlush,,1733154398173.2d5146d261fcea0ac5ee2aef06045ae2. at 1733154407606Flushing 2d5146d261fcea0ac5ee2aef06045ae2/a: creating writer at 1733154407606Flushing 2d5146d261fcea0ac5ee2aef06045ae2/a: appending metadata at 1733154407619 (+13 ms)Flushing 2d5146d261fcea0ac5ee2aef06045ae2/a: closing flushed file at 1733154407619Flushing 2d5146d261fcea0ac5ee2aef06045ae2/b: creating writer at 1733154407633 (+14 ms)Flushing 2d5146d261fcea0ac5ee2aef06045ae2/b: appending metadata at 1733154407652 (+19 ms)Flushing 2d5146d261fcea0ac5ee2aef06045ae2/b: closing flushed file at 1733154407652Flushing 2d5146d261fcea0ac5ee2aef06045ae2/c: creating writer at 1733154407666 (+14 ms)Flushing 2d5146d261fcea0ac5ee2aef06045ae2/c: appending metadata at 1733154407679 (+13 ms)Flushing 2d5146d261fcea0ac5ee2aef06045ae2/c: closing flushed file at 1733154407679Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aa14761: reopening flushed file at 1733154407693 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6268dd81: reopening flushed file at 1733154407701 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@673ee18b: reopening flushed file at 1733154407707 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 2d5146d261fcea0ac5ee2aef06045ae2 in 107ms, sequenceid=26, compaction requested=false; wal=null at 1733154407713 (+6 ms)Cleaning up temporary data from old regions at 1733154407715 (+2 ms)Running coprocessor post-open hooks at 1733154407720 (+5 ms)Region opened successfully at 1733154407720 2024-12-02T15:46:47,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414 (was 414), OpenFileDescriptor=1180 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=9693 (was 9750) 2024-12-02T15:46:47,740 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-02T15:46:47,751 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=414, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=9692 2024-12-02T15:46:47,751 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-02T15:46:47,763 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:47,765 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:47,766 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:47,768 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-18578102, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-18578102, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:47,779 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-18578102/hregion-18578102.1733154407769, exclude list is [], retry=0 2024-12-02T15:46:47,782 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:47,782 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:47,782 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:47,806 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-18578102/hregion-18578102.1733154407769 2024-12-02T15:46:47,806 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:47,806 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6e84389563b7dfa5ee9592a40621dc72, NAME => 'testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741915_1093 (size=61) 2024-12-02T15:46:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741915_1093 (size=61) 2024-12-02T15:46:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741915_1093 (size=61) 2024-12-02T15:46:47,816 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:47,818 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,819 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e84389563b7dfa5ee9592a40621dc72 columnFamilyName a 2024-12-02T15:46:47,819 DEBUG [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:47,820 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(327): Store=6e84389563b7dfa5ee9592a40621dc72/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:47,820 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,820 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,821 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,821 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,821 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,825 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:47,825 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6e84389563b7dfa5ee9592a40621dc72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59296433, jitterRate=-0.11641429364681244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:47,826 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6e84389563b7dfa5ee9592a40621dc72: Writing region info on filesystem at 1733154407816Initializing all the Stores at 1733154407817 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154407817Cleaning up temporary data from old regions at 1733154407821 (+4 ms)Region opened successfully at 1733154407826 (+5 ms) 2024-12-02T15:46:47,826 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6e84389563b7dfa5ee9592a40621dc72, disabling compactions & flushes 2024-12-02T15:46:47,827 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,827 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,827 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. after waiting 0 ms 2024-12-02T15:46:47,827 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,827 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,827 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6e84389563b7dfa5ee9592a40621dc72: Waiting for close lock at 1733154407826Disabling compacts and flushes for region at 1733154407826Disabling writes for close at 1733154407827 (+1 ms)Writing region close event to WAL at 1733154407827Closed at 1733154407827 2024-12-02T15:46:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741914_1092 (size=175) 2024-12-02T15:46:47,830 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-18578102/hregion-18578102.1733154407769 not finished, retry = 0 2024-12-02T15:46:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741914_1092 (size=175) 2024-12-02T15:46:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741914_1092 (size=175) 2024-12-02T15:46:47,934 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:47,934 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-18578102:(num 1733154407769) 2024-12-02T15:46:47,935 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:47,938 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:47,956 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939, exclude list is [], retry=0 2024-12-02T15:46:47,958 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:47,958 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:47,958 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:47,979 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 2024-12-02T15:46:47,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e84389563b7dfa5ee9592a40621dc72, NAME => 'testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,980 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,981 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,982 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e84389563b7dfa5ee9592a40621dc72 columnFamilyName a 2024-12-02T15:46:47,982 DEBUG [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:47,982 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(327): Store=6e84389563b7dfa5ee9592a40621dc72/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:47,982 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,984 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,984 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,984 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,986 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:47,986 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6e84389563b7dfa5ee9592a40621dc72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67674585, jitterRate=0.00842989981174469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:47,987 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6e84389563b7dfa5ee9592a40621dc72: Writing region info on filesystem at 1733154407980Initializing all the Stores at 1733154407981 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154407981Cleaning up temporary data from old regions at 1733154407984 (+3 ms)Region opened successfully at 1733154407987 (+3 ms) 2024-12-02T15:46:47,996 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6e84389563b7dfa5ee9592a40621dc72, disabling compactions & flushes 2024-12-02T15:46:47,996 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,996 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,996 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. after waiting 0 ms 2024-12-02T15:46:47,996 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,997 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,997 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. 2024-12-02T15:46:47,997 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6e84389563b7dfa5ee9592a40621dc72: Waiting for close lock at 1733154407996Disabling compacts and flushes for region at 1733154407996Disabling writes for close at 1733154407996Writing region close event to WAL at 1733154407997 (+1 ms)Closed at 1733154407997 2024-12-02T15:46:48,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741916_1094 (size=1795) 2024-12-02T15:46:48,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741916_1094 (size=1795) 2024-12-02T15:46:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741916_1094 (size=1795) 2024-12-02T15:46:48,013 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939, size=1.8 K (1795bytes) 2024-12-02T15:46:48,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 2024-12-02T15:46:48,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 after 0ms 2024-12-02T15:46:48,036 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:48,036 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 took 24ms 2024-12-02T15:46:48,038 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 so closing down 2024-12-02T15:46:48,038 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:48,040 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733154407939.temp 2024-12-02T15:46:48,064 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000003-wal.1733154407939.temp 2024-12-02T15:46:48,064 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741917_1095 (size=1795) 2024-12-02T15:46:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741917_1095 (size=1795) 2024-12-02T15:46:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741917_1095 (size=1795) 2024-12-02T15:46:48,072 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000003-wal.1733154407939.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:48,073 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000003-wal.1733154407939.temp to hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000012 2024-12-02T15:46:48,073 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-02T15:46:48,073 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939, journal: Splitting hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939, size=1.8 K (1795bytes) at 1733154408013Finishing writing output for hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 so closing down at 1733154408038 (+25 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000003-wal.1733154407939.temp at 1733154408064 (+26 ms)3 split writer threads finished at 1733154408064Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000003-wal.1733154407939.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733154408072 (+8 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000003-wal.1733154407939.temp to hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000012 at 1733154408073 (+1 ms)Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733154408073 2024-12-02T15:46:48,075 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154407939 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154407939 2024-12-02T15:46:48,076 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000012 2024-12-02T15:46:48,101 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:48,425 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:48,427 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:48,437 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154408427, exclude list is [], retry=0 2024-12-02T15:46:48,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:48,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:48,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:48,463 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154408427 2024-12-02T15:46:48,463 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:48,463 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e84389563b7dfa5ee9592a40621dc72, NAME => 'testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:48,464 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:48,464 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,464 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,466 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,468 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e84389563b7dfa5ee9592a40621dc72 columnFamilyName a 2024-12-02T15:46:48,468 DEBUG [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,469 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(327): Store=6e84389563b7dfa5ee9592a40621dc72/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,469 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,470 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,471 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000012 2024-12-02T15:46:48,493 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:48,494 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000012 2024-12-02T15:46:48,495 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6e84389563b7dfa5ee9592a40621dc72 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-02T15:46:48,510 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/.tmp/a/e3b86fe73da24268a0b9bac41116ca77 is 79, key is testDatalossWhenInputError/a:x0/1733154407987/Put/seqid=0 2024-12-02T15:46:48,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741919_1097 (size=5808) 2024-12-02T15:46:48,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741919_1097 (size=5808) 2024-12-02T15:46:48,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741919_1097 (size=5808) 2024-12-02T15:46:48,517 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/.tmp/a/e3b86fe73da24268a0b9bac41116ca77 2024-12-02T15:46:48,526 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/.tmp/a/e3b86fe73da24268a0b9bac41116ca77 as hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/a/e3b86fe73da24268a0b9bac41116ca77 2024-12-02T15:46:48,534 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/a/e3b86fe73da24268a0b9bac41116ca77, entries=10, sequenceid=12, filesize=5.7 K 2024-12-02T15:46:48,534 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6e84389563b7dfa5ee9592a40621dc72 in 40ms, sequenceid=12, compaction requested=false; wal=null 2024-12-02T15:46:48,535 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/0000000000000000012 2024-12-02T15:46:48,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,537 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,540 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,542 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-02T15:46:48,543 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6e84389563b7dfa5ee9592a40621dc72; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70205047, jitterRate=0.04613672196865082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:48,544 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6e84389563b7dfa5ee9592a40621dc72: Writing region info on filesystem at 1733154408464Initializing all the Stores at 1733154408466 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408466Obtaining lock to block concurrent updates at 1733154408495 (+29 ms)Preparing flush snapshotting stores in 6e84389563b7dfa5ee9592a40621dc72 at 1733154408495Finished memstore snapshotting testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733154408495Flushing stores of testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72. at 1733154408495Flushing 6e84389563b7dfa5ee9592a40621dc72/a: creating writer at 1733154408495Flushing 6e84389563b7dfa5ee9592a40621dc72/a: appending metadata at 1733154408510 (+15 ms)Flushing 6e84389563b7dfa5ee9592a40621dc72/a: closing flushed file at 1733154408510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b17cafb: reopening flushed file at 1733154408524 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6e84389563b7dfa5ee9592a40621dc72 in 40ms, sequenceid=12, compaction requested=false; wal=null at 1733154408534 (+10 ms)Cleaning up temporary data from old regions at 1733154408537 (+3 ms)Region opened successfully at 1733154408544 (+7 ms) 2024-12-02T15:46:48,548 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e84389563b7dfa5ee9592a40621dc72, NAME => 'testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:48,548 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733154407764.6e84389563b7dfa5ee9592a40621dc72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:48,548 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,548 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,549 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,550 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e84389563b7dfa5ee9592a40621dc72 columnFamilyName a 2024-12-02T15:46:48,550 DEBUG [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,557 DEBUG [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/a/e3b86fe73da24268a0b9bac41116ca77 2024-12-02T15:46:48,557 INFO [StoreOpener-6e84389563b7dfa5ee9592a40621dc72-1 {}] regionserver.HStore(327): Store=6e84389563b7dfa5ee9592a40621dc72/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,557 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,560 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,560 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,561 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6e84389563b7dfa5ee9592a40621dc72 2024-12-02T15:46:48,564 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testDatalossWhenInputError/6e84389563b7dfa5ee9592a40621dc72/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-02T15:46:48,565 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6e84389563b7dfa5ee9592a40621dc72; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72795163, jitterRate=0.08473245799541473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T15:46:48,565 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6e84389563b7dfa5ee9592a40621dc72: Writing region info on filesystem at 1733154408548Initializing all the Stores at 1733154408549 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408549Cleaning up temporary data from old regions at 1733154408560 (+11 ms)Region opened successfully at 1733154408565 (+5 ms) 2024-12-02T15:46:48,580 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=424 (was 414) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:49504 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:43412 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:49934 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:58450 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1466660644-172.17.0.3-1733154372547:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:56746 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 339), ProcessCount=11 (was 11), AvailableMemoryMB=9687 (was 9692) 2024-12-02T15:46:48,580 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-02T15:46:48,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=9687 2024-12-02T15:46:48,591 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-02T15:46:48,604 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:48,605 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:48,606 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:48,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37601453, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-37601453, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:48,618 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37601453/hregion-37601453.1733154408608, exclude list is [], retry=0 2024-12-02T15:46:48,621 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:48,621 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:48,621 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:48,645 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37601453/hregion-37601453.1733154408608 2024-12-02T15:46:48,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:48,646 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1b2eaf34d161dab51d8921979503d5e6, NAME => 'testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741921_1099 (size=63) 2024-12-02T15:46:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741921_1099 (size=63) 2024-12-02T15:46:48,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741921_1099 (size=63) 2024-12-02T15:46:48,655 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:48,656 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,657 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName a 2024-12-02T15:46:48,658 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,658 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,658 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,660 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName b 2024-12-02T15:46:48,660 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,660 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,660 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,662 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName c 2024-12-02T15:46:48,662 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,662 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,663 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,664 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,665 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,665 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,665 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:48,667 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,669 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:48,670 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b2eaf34d161dab51d8921979503d5e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74836173, jitterRate=0.11514587700366974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:48,670 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b2eaf34d161dab51d8921979503d5e6: Writing region info on filesystem at 1733154408655Initializing all the Stores at 1733154408656 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408656Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408656Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408656Cleaning up temporary data from old regions at 1733154408665 (+9 ms)Region opened successfully at 1733154408670 (+5 ms) 2024-12-02T15:46:48,671 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b2eaf34d161dab51d8921979503d5e6, disabling compactions & flushes 2024-12-02T15:46:48,671 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:48,671 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:48,671 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. after waiting 0 ms 2024-12-02T15:46:48,671 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:48,671 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:48,671 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b2eaf34d161dab51d8921979503d5e6: Waiting for close lock at 1733154408671Disabling compacts and flushes for region at 1733154408671Disabling writes for close at 1733154408671Writing region close event to WAL at 1733154408671Closed at 1733154408671 2024-12-02T15:46:48,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741920_1098 (size=175) 2024-12-02T15:46:48,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741920_1098 (size=175) 2024-12-02T15:46:48,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741920_1098 (size=175) 2024-12-02T15:46:48,677 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:48,677 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37601453:(num 1733154408608) 2024-12-02T15:46:48,677 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:48,678 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:48,690 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679, exclude list is [], retry=0 2024-12-02T15:46:48,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:48,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:48,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:48,715 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 2024-12-02T15:46:48,716 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b2eaf34d161dab51d8921979503d5e6, NAME => 'testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,718 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,719 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName a 2024-12-02T15:46:48,719 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,719 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,719 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,720 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName b 2024-12-02T15:46:48,720 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,721 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,721 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,722 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName c 2024-12-02T15:46:48,722 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:48,722 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:48,722 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,723 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,724 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,725 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,725 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,726 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:48,727 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:48,728 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b2eaf34d161dab51d8921979503d5e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67490506, jitterRate=0.005686908960342407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:48,729 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b2eaf34d161dab51d8921979503d5e6: Writing region info on filesystem at 1733154408716Initializing all the Stores at 1733154408717 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408717Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408717Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154408717Cleaning up temporary data from old regions at 1733154408725 (+8 ms)Region opened successfully at 1733154408729 (+4 ms) 2024-12-02T15:46:48,733 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733154408732/Put/seqid=0 2024-12-02T15:46:48,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741923_1101 (size=4875) 2024-12-02T15:46:48,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741923_1101 (size=4875) 2024-12-02T15:46:48,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741923_1101 (size=4875) 2024-12-02T15:46:48,744 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733154408743/Put/seqid=0 2024-12-02T15:46:48,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741924_1102 (size=4875) 2024-12-02T15:46:48,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741924_1102 (size=4875) 2024-12-02T15:46:48,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741924_1102 (size=4875) 2024-12-02T15:46:48,754 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733154408754/Put/seqid=0 2024-12-02T15:46:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741925_1103 (size=4875) 2024-12-02T15:46:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741925_1103 (size=4875) 2024-12-02T15:46:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741925_1103 (size=4875) 2024-12-02T15:46:48,761 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 1b2eaf34d161dab51d8921979503d5e6/a 2024-12-02T15:46:48,765 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-02T15:46:48,765 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T15:46:48,765 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 1b2eaf34d161dab51d8921979503d5e6/a 2024-12-02T15:46:48,768 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-02T15:46:48,768 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T15:46:48,768 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 1b2eaf34d161dab51d8921979503d5e6/a 2024-12-02T15:46:48,771 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-02T15:46:48,771 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-02T15:46:48,771 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b2eaf34d161dab51d8921979503d5e6 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-02T15:46:48,785 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/.tmp/a/0b0faa5489ae4f76b278b0a19d6624b4 is 55, key is testCompactedBulkLoadedFiles/a:a/1733154408729/Put/seqid=0 2024-12-02T15:46:48,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741926_1104 (size=5107) 2024-12-02T15:46:48,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741926_1104 (size=5107) 2024-12-02T15:46:48,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741926_1104 (size=5107) 2024-12-02T15:46:48,793 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/.tmp/a/0b0faa5489ae4f76b278b0a19d6624b4 2024-12-02T15:46:48,800 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/.tmp/a/0b0faa5489ae4f76b278b0a19d6624b4 as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4 2024-12-02T15:46:48,805 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4, entries=1, sequenceid=4, filesize=5.0 K 2024-12-02T15:46:48,806 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 1b2eaf34d161dab51d8921979503d5e6 in 35ms, sequenceid=4, compaction requested=false 2024-12-02T15:46:48,806 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1b2eaf34d161dab51d8921979503d5e6: 2024-12-02T15:46:48,808 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ 2024-12-02T15:46:48,809 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ 2024-12-02T15:46:48,810 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ 2024-12-02T15:46:48,811 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile0 into 1b2eaf34d161dab51d8921979503d5e6/a as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ - updating store file list. 2024-12-02T15:46:48,815 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 156151b36d3d424d9f4176e3d8b63780_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:48,816 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ into 1b2eaf34d161dab51d8921979503d5e6/a 2024-12-02T15:46:48,816 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile0 into 1b2eaf34d161dab51d8921979503d5e6/a (new location: hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_) 2024-12-02T15:46:48,817 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile1 into 1b2eaf34d161dab51d8921979503d5e6/a as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ - updating store file list. 2024-12-02T15:46:48,821 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1de624814ba04747be22d8a640279fff_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:48,821 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ into 1b2eaf34d161dab51d8921979503d5e6/a 2024-12-02T15:46:48,821 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile1 into 1b2eaf34d161dab51d8921979503d5e6/a (new location: hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_) 2024-12-02T15:46:48,822 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile2 into 1b2eaf34d161dab51d8921979503d5e6/a as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ - updating store file list. 2024-12-02T15:46:48,826 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:48,826 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ into 1b2eaf34d161dab51d8921979503d5e6/a 2024-12-02T15:46:48,826 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33469/hbase/testCompactedBulkLoadedFiles/hfile2 into 1b2eaf34d161dab51d8921979503d5e6/a (new location: hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_) 2024-12-02T15:46:48,832 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T15:46:48,832 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1b2eaf34d161dab51d8921979503d5e6/a is initiating major compaction (all files) 2024-12-02T15:46:48,832 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1b2eaf34d161dab51d8921979503d5e6/a in testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:48,832 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_] into tmpdir=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/.tmp, totalSize=19.3 K 2024-12-02T15:46:48,833 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0b0faa5489ae4f76b278b0a19d6624b4, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733154408729 2024-12-02T15:46:48,833 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 156151b36d3d424d9f4176e3d8b63780_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T15:46:48,834 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1de624814ba04747be22d8a640279fff_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T15:46:48,834 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-02T15:46:48,845 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/.tmp/a/f7bd170fd61a48c5bb614b3c6aa7a8aa is 55, key is testCompactedBulkLoadedFiles/a:a/1733154408729/Put/seqid=0 2024-12-02T15:46:48,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741927_1105 (size=6154) 2024-12-02T15:46:48,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741927_1105 (size=6154) 2024-12-02T15:46:48,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741927_1105 (size=6154) 2024-12-02T15:46:48,858 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/.tmp/a/f7bd170fd61a48c5bb614b3c6aa7a8aa as hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/f7bd170fd61a48c5bb614b3c6aa7a8aa 2024-12-02T15:46:48,867 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 1b2eaf34d161dab51d8921979503d5e6/a of 1b2eaf34d161dab51d8921979503d5e6 into f7bd170fd61a48c5bb614b3c6aa7a8aa(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T15:46:48,867 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1b2eaf34d161dab51d8921979503d5e6: 2024-12-02T15:46:48,867 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-02T15:46:48,867 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-02T15:46:48,897 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679, size=0 (0bytes) 2024-12-02T15:46:48,898 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 might be still open, length is 0 2024-12-02T15:46:48,898 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 2024-12-02T15:46:48,898 WARN [IPC Server handler 2 on default port 33469 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-02T15:46:48,898 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 after 0ms 2024-12-02T15:46:51,437 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:43446 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43446 dst: /127.0.0.1:38685 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38685 remote=/127.0.0.1:43446]. Total timeout mills is 60000, 57431 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:51,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:49974 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:42367:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49974 dst: /127.0.0.1:42367 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:51,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:56768 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43105:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56768 dst: /127.0.0.1:43105 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:51,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741922_1106 (size=1794) 2024-12-02T15:46:51,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741922_1106 (size=1794) 2024-12-02T15:46:51,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741922_1106 (size=1794) 2024-12-02T15:46:52,900 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 after 4002ms 2024-12-02T15:46:52,929 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T15:46:52,931 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:52,932 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 took 4034ms 2024-12-02T15:46:52,934 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679; continuing. 2024-12-02T15:46:52,934 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 so closing down 2024-12-02T15:46:52,934 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:52,935 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733154408679.temp 2024-12-02T15:46:52,967 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000003-wal.1733154408679.temp 2024-12-02T15:46:52,967 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:52,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741928_1107 (size=800) 2024-12-02T15:46:52,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741928_1107 (size=800) 2024-12-02T15:46:52,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741928_1107 (size=800) 2024-12-02T15:46:52,974 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000003-wal.1733154408679.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:52,976 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000003-wal.1733154408679.temp to hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000008 2024-12-02T15:46:52,976 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 44 ms; skipped=3; WAL=hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T15:46:52,976 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679, journal: Splitting hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679, size=0 (0bytes) at 1733154408897Finishing writing output for hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 so closing down at 1733154412934 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000003-wal.1733154408679.temp at 1733154412967 (+33 ms)3 split writer threads finished at 1733154412967Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000003-wal.1733154408679.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733154412974 (+7 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000003-wal.1733154408679.temp to hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000008 at 1733154412976 (+2 ms)Processed 5 edits across 1 Regions in 44 ms; skipped=3; WAL=hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679, size=0, length=0, corrupted=false, cancelled=false at 1733154412976 2024-12-02T15:46:52,977 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154408679 2024-12-02T15:46:52,978 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000008 2024-12-02T15:46:52,978 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:52,980 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:52,991 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154412980, exclude list is [], retry=0 2024-12-02T15:46:52,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:52,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:52,994 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:53,015 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154412980 2024-12-02T15:46:53,015 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449)] 2024-12-02T15:46:53,016 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b2eaf34d161dab51d8921979503d5e6, NAME => 'testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:53,016 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:53,016 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,016 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,017 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,018 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName a 2024-12-02T15:46:53,018 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,026 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4 2024-12-02T15:46:53,030 DEBUG [StoreFileOpener-1b2eaf34d161dab51d8921979503d5e6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 156151b36d3d424d9f4176e3d8b63780_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:53,030 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ 2024-12-02T15:46:53,034 DEBUG [StoreFileOpener-1b2eaf34d161dab51d8921979503d5e6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1de624814ba04747be22d8a640279fff_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:53,034 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ 2024-12-02T15:46:53,038 DEBUG [StoreFileOpener-1b2eaf34d161dab51d8921979503d5e6-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-02T15:46:53,038 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ 2024-12-02T15:46:53,042 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/f7bd170fd61a48c5bb614b3c6aa7a8aa 2024-12-02T15:46:53,043 WARN [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@546df373 2024-12-02T15:46:53,043 WARN [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@546df373 2024-12-02T15:46:53,043 WARN [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@546df373 2024-12-02T15:46:53,043 WARN [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@546df373 2024-12-02T15:46:53,043 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_] to archive 2024-12-02T15:46:53,044 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T15:46:53,045 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4 to hdfs://localhost:33469/hbase/archive/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/0b0faa5489ae4f76b278b0a19d6624b4 2024-12-02T15:46:53,047 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ to hdfs://localhost:33469/hbase/archive/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/156151b36d3d424d9f4176e3d8b63780_SeqId_4_ 2024-12-02T15:46:53,048 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ to hdfs://localhost:33469/hbase/archive/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/1de624814ba04747be22d8a640279fff_SeqId_4_ 2024-12-02T15:46:53,049 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ to hdfs://localhost:33469/hbase/archive/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/a/822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_ 2024-12-02T15:46:53,049 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,049 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,050 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName b 2024-12-02T15:46:53,050 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,050 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,050 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,051 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b2eaf34d161dab51d8921979503d5e6 columnFamilyName c 2024-12-02T15:46:53,051 DEBUG [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,052 INFO [StoreOpener-1b2eaf34d161dab51d8921979503d5e6-1 {}] regionserver.HStore(327): Store=1b2eaf34d161dab51d8921979503d5e6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,052 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,053 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,055 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,055 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000008 2024-12-02T15:46:53,079 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:53,081 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 1b2eaf34d161dab51d8921979503d5e6 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "1b2eaf34d161dab51d8921979503d5e6" family_name: "a" compaction_input: "0b0faa5489ae4f76b278b0a19d6624b4" compaction_input: "156151b36d3d424d9f4176e3d8b63780_SeqId_4_" compaction_input: "1de624814ba04747be22d8a640279fff_SeqId_4_" compaction_input: "822a15e7e2fe4b17896cbcfca049cc14_SeqId_4_" compaction_output: "f7bd170fd61a48c5bb614b3c6aa7a8aa" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-02T15:46:53,081 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-02T15:46:53,081 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000008 2024-12-02T15:46:53,082 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/0000000000000000008 2024-12-02T15:46:53,083 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,083 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,084 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:53,085 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1b2eaf34d161dab51d8921979503d5e6 2024-12-02T15:46:53,087 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testCompactedBulkLoadedFiles/1b2eaf34d161dab51d8921979503d5e6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-02T15:46:53,088 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1b2eaf34d161dab51d8921979503d5e6; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65532386, jitterRate=-0.023491352796554565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:53,089 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1b2eaf34d161dab51d8921979503d5e6: Writing region info on filesystem at 1733154413016Initializing all the Stores at 1733154413017 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413017Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413017Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413017Cleaning up temporary data from old regions at 1733154413083 (+66 ms)Region opened successfully at 1733154413089 (+6 ms) 2024-12-02T15:46:53,091 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1b2eaf34d161dab51d8921979503d5e6, disabling compactions & flushes 2024-12-02T15:46:53,092 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:53,092 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:53,092 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. after waiting 0 ms 2024-12-02T15:46:53,092 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:53,093 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733154408604.1b2eaf34d161dab51d8921979503d5e6. 2024-12-02T15:46:53,093 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1b2eaf34d161dab51d8921979503d5e6: Waiting for close lock at 1733154413091Disabling compacts and flushes for region at 1733154413091Disabling writes for close at 1733154413092 (+1 ms)Writing region close event to WAL at 1733154413093 (+1 ms)Closed at 1733154413093 2024-12-02T15:46:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741929_1108 (size=175) 2024-12-02T15:46:53,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741929_1108 (size=175) 2024-12-02T15:46:53,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741929_1108 (size=175) 2024-12-02T15:46:53,098 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:53,099 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733154412980) 2024-12-02T15:46:53,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=437 (was 424) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-828355161_22 at /127.0.0.1:56838 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33469 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-828355161_22 at /127.0.0.1:50044 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33469 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-828355161_22 at /127.0.0.1:43514 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 339), ProcessCount=11 (was 11), AvailableMemoryMB=9694 (was 9687) - AvailableMemoryMB LEAK? - 2024-12-02T15:46:53,111 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-12-02T15:46:53,123 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=9692 2024-12-02T15:46:53,123 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-12-02T15:46:53,135 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:53,137 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T15:46:53,137 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T15:46:53,140 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82641965, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/hregion-82641965, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:53,150 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82641965/hregion-82641965.1733154413140, exclude list is [], retry=0 2024-12-02T15:46:53,153 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:53,153 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:53,154 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:53,177 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82641965/hregion-82641965.1733154413140 2024-12-02T15:46:53,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:53,178 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c1d5704228f8da4dce8e669078aa110d, NAME => 'testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33469/hbase 2024-12-02T15:46:53,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741931_1110 (size=67) 2024-12-02T15:46:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741931_1110 (size=67) 2024-12-02T15:46:53,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741931_1110 (size=67) 2024-12-02T15:46:53,186 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:53,187 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,188 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName a 2024-12-02T15:46:53,189 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,189 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,189 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,191 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName b 2024-12-02T15:46:53,191 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,191 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,191 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,193 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName c 2024-12-02T15:46:53,193 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,193 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,193 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,195 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,195 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,196 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:53,197 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,199 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T15:46:53,199 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1d5704228f8da4dce8e669078aa110d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71556075, jitterRate=0.06626860797405243}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:53,199 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1d5704228f8da4dce8e669078aa110d: Writing region info on filesystem at 1733154413186Initializing all the Stores at 1733154413187 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413187Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413187Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413187Cleaning up temporary data from old regions at 1733154413195 (+8 ms)Region opened successfully at 1733154413199 (+4 ms) 2024-12-02T15:46:53,199 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1d5704228f8da4dce8e669078aa110d, disabling compactions & flushes 2024-12-02T15:46:53,199 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,200 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,200 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. after waiting 0 ms 2024-12-02T15:46:53,200 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,200 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,200 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1d5704228f8da4dce8e669078aa110d: Waiting for close lock at 1733154413199Disabling compacts and flushes for region at 1733154413199Disabling writes for close at 1733154413200 (+1 ms)Writing region close event to WAL at 1733154413200Closed at 1733154413200 2024-12-02T15:46:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741930_1109 (size=175) 2024-12-02T15:46:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741930_1109 (size=175) 2024-12-02T15:46:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741930_1109 (size=175) 2024-12-02T15:46:53,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:53,208 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82641965:(num 1733154413140) 2024-12-02T15:46:53,208 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:53,210 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:53,221 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210, exclude list is [], retry=0 2024-12-02T15:46:53,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:53,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:53,225 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:53,249 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 2024-12-02T15:46:53,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33971:33971),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763)] 2024-12-02T15:46:53,250 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1d5704228f8da4dce8e669078aa110d, NAME => 'testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:53,250 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:53,250 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,250 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,251 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,252 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName a 2024-12-02T15:46:53,252 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,253 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,253 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,254 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName b 2024-12-02T15:46:53,254 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,254 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,254 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,255 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName c 2024-12-02T15:46:53,255 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,255 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,255 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,256 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,257 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,258 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,258 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,259 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:53,260 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,260 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1d5704228f8da4dce8e669078aa110d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70498256, jitterRate=0.050505876541137695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:53,261 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1d5704228f8da4dce8e669078aa110d: Writing region info on filesystem at 1733154413250Initializing all the Stores at 1733154413251 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413251Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413251Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413251Cleaning up temporary data from old regions at 1733154413258 (+7 ms)Region opened successfully at 1733154413261 (+3 ms) 2024-12-02T15:46:53,268 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1d5704228f8da4dce8e669078aa110d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-02T15:46:53,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/a/413067cf1549490090f785324e25f92e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733154413261/Put/seqid=0 2024-12-02T15:46:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741933_1112 (size=5958) 2024-12-02T15:46:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741933_1112 (size=5958) 2024-12-02T15:46:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741933_1112 (size=5958) 2024-12-02T15:46:53,288 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/a/413067cf1549490090f785324e25f92e 2024-12-02T15:46:53,293 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/a/413067cf1549490090f785324e25f92e as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/a/413067cf1549490090f785324e25f92e 2024-12-02T15:46:53,299 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/a/413067cf1549490090f785324e25f92e, entries=10, sequenceid=13, filesize=5.8 K 2024-12-02T15:46:53,300 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c1d5704228f8da4dce8e669078aa110d in 32ms, sequenceid=13, compaction requested=false 2024-12-02T15:46:53,300 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c1d5704228f8da4dce8e669078aa110d: 2024-12-02T15:46:53,315 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1d5704228f8da4dce8e669078aa110d, disabling compactions & flushes 2024-12-02T15:46:53,315 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,315 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,315 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. after waiting 0 ms 2024-12-02T15:46:53,315 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,316 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,316 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:53,316 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1d5704228f8da4dce8e669078aa110d: Waiting for close lock at 1733154413315Disabling compacts and flushes for region at 1733154413315Disabling writes for close at 1733154413315Writing region close event to WAL at 1733154413315Closed at 1733154413316 (+1 ms) 2024-12-02T15:46:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741932_1111 (size=6243) 2024-12-02T15:46:53,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741932_1111 (size=6243) 2024-12-02T15:46:53,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741932_1111 (size=6243) 2024-12-02T15:46:53,332 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210, size=6.1 K (6243bytes) 2024-12-02T15:46:53,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 2024-12-02T15:46:53,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 after 0ms 2024-12-02T15:46:53,354 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:53,355 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 took 23ms 2024-12-02T15:46:53,356 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 so closing down 2024-12-02T15:46:53,356 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:53,358 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733154413210.temp 2024-12-02T15:46:53,381 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000003-wal.1733154413210.temp 2024-12-02T15:46:53,381 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:53,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741934_1113 (size=5575) 2024-12-02T15:46:53,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741934_1113 (size=5575) 2024-12-02T15:46:53,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741934_1113 (size=5575) 2024-12-02T15:46:53,387 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000003-wal.1733154413210.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:53,388 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000003-wal.1733154413210.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000035 2024-12-02T15:46:53,388 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 33 ms; skipped=2; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-02T15:46:53,388 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210, journal: Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210, size=6.1 K (6243bytes) at 1733154413332Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 so closing down at 1733154413356 (+24 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000003-wal.1733154413210.temp at 1733154413381 (+25 ms)3 split writer threads finished at 1733154413381Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000003-wal.1733154413210.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733154413387 (+6 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000003-wal.1733154413210.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000035 at 1733154413388 (+1 ms)Processed 32 edits across 1 Regions in 33 ms; skipped=2; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733154413388 2024-12-02T15:46:53,389 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413210 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154413210 2024-12-02T15:46:53,390 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000035 2024-12-02T15:46:53,390 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:53,392 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:53,403 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392, exclude list is [], retry=0 2024-12-02T15:46:53,405 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:53,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:53,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:53,432 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 2024-12-02T15:46:53,432 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:53,432 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1d5704228f8da4dce8e669078aa110d, NAME => 'testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T15:46:53,432 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:53,433 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,433 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,434 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,435 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName a 2024-12-02T15:46:53,435 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,440 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/a/413067cf1549490090f785324e25f92e 2024-12-02T15:46:53,440 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,440 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,441 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName b 2024-12-02T15:46:53,441 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,442 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,442 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,442 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName c 2024-12-02T15:46:53,443 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:53,443 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:53,443 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,445 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,446 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000035 2024-12-02T15:46:53,468 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:53,470 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000035 2024-12-02T15:46:53,470 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1d5704228f8da4dce8e669078aa110d 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-02T15:46:53,483 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/b/12d6f523faad4aad9bbf704e8f724196 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733154413300/Put/seqid=0 2024-12-02T15:46:53,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741936_1115 (size=5958) 2024-12-02T15:46:53,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741936_1115 (size=5958) 2024-12-02T15:46:53,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741936_1115 (size=5958) 2024-12-02T15:46:53,489 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/b/12d6f523faad4aad9bbf704e8f724196 2024-12-02T15:46:53,508 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/c/b34be5bf3fc34e1ebd8c22f633a04f55 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733154413306/Put/seqid=0 2024-12-02T15:46:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741937_1116 (size=5958) 2024-12-02T15:46:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741937_1116 (size=5958) 2024-12-02T15:46:53,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741937_1116 (size=5958) 2024-12-02T15:46:53,514 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/c/b34be5bf3fc34e1ebd8c22f633a04f55 2024-12-02T15:46:53,518 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/b/12d6f523faad4aad9bbf704e8f724196 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/b/12d6f523faad4aad9bbf704e8f724196 2024-12-02T15:46:53,522 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/b/12d6f523faad4aad9bbf704e8f724196, entries=10, sequenceid=35, filesize=5.8 K 2024-12-02T15:46:53,523 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/c/b34be5bf3fc34e1ebd8c22f633a04f55 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/c/b34be5bf3fc34e1ebd8c22f633a04f55 2024-12-02T15:46:53,529 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/c/b34be5bf3fc34e1ebd8c22f633a04f55, entries=10, sequenceid=35, filesize=5.8 K 2024-12-02T15:46:53,529 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c1d5704228f8da4dce8e669078aa110d in 59ms, sequenceid=35, compaction requested=false; wal=null 2024-12-02T15:46:53,530 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000035 2024-12-02T15:46:53,531 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,531 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,531 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:53,533 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:53,535 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-02T15:46:53,536 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1d5704228f8da4dce8e669078aa110d; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62628209, jitterRate=-0.06676696240901947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:53,536 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1d5704228f8da4dce8e669078aa110d: Writing region info on filesystem at 1733154413433Initializing all the Stores at 1733154413433Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413434 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413434Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154413434Obtaining lock to block concurrent updates at 1733154413470 (+36 ms)Preparing flush snapshotting stores in c1d5704228f8da4dce8e669078aa110d at 1733154413470Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733154413470Flushing stores of testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. at 1733154413470Flushing c1d5704228f8da4dce8e669078aa110d/b: creating writer at 1733154413470Flushing c1d5704228f8da4dce8e669078aa110d/b: appending metadata at 1733154413482 (+12 ms)Flushing c1d5704228f8da4dce8e669078aa110d/b: closing flushed file at 1733154413482Flushing c1d5704228f8da4dce8e669078aa110d/c: creating writer at 1733154413493 (+11 ms)Flushing c1d5704228f8da4dce8e669078aa110d/c: appending metadata at 1733154413507 (+14 ms)Flushing c1d5704228f8da4dce8e669078aa110d/c: closing flushed file at 1733154413507Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb60a7b: reopening flushed file at 1733154413517 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e455269: reopening flushed file at 1733154413522 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c1d5704228f8da4dce8e669078aa110d in 59ms, sequenceid=35, compaction requested=false; wal=null at 1733154413529 (+7 ms)Cleaning up temporary data from old regions at 1733154413531 (+2 ms)Region opened successfully at 1733154413536 (+5 ms) 2024-12-02T15:46:53,615 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392, size=0 (0bytes) 2024-12-02T15:46:53,615 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 might be still open, length is 0 2024-12-02T15:46:53,615 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 2024-12-02T15:46:53,616 WARN [IPC Server handler 3 on default port 33469 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-02T15:46:53,616 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 after 1ms 2024-12-02T15:46:54,437 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:56902 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43105:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56902 dst: /127.0.0.1:43105 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43105 remote=/127.0.0.1:56902]. Total timeout mills is 60000, 59131 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:54,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:43548 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38685:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43548 dst: /127.0.0.1:38685 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:54,438 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1260188680_22 at /127.0.0.1:50088 [Receiving block BP-1466660644-172.17.0.3-1733154372547:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:42367:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50088 dst: /127.0.0.1:42367 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:54,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741935_1117 (size=5567) 2024-12-02T15:46:54,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741935_1117 (size=5567) 2024-12-02T15:46:54,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741935_1117 (size=5567) 2024-12-02T15:46:56,606 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-02T15:46:56,607 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:56,609 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-02T15:46:56,609 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-02T15:46:57,618 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 after 4003ms 2024-12-02T15:46:57,648 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:57,648 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 took 4033ms 2024-12-02T15:46:57,650 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392; continuing. 2024-12-02T15:46:57,650 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 so closing down 2024-12-02T15:46:57,650 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T15:46:57,651 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733154413392.temp 2024-12-02T15:46:57,672 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000037-wal.1733154413392.temp 2024-12-02T15:46:57,673 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T15:46:57,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741938_1118 (size=5575) 2024-12-02T15:46:57,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741938_1118 (size=5575) 2024-12-02T15:46:57,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741938_1118 (size=5575) 2024-12-02T15:46:57,679 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000037-wal.1733154413392.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-02T15:46:57,680 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000037-wal.1733154413392.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000066 2024-12-02T15:46:57,680 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392, size=0, length=0, corrupted=false, cancelled=false 2024-12-02T15:46:57,680 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392, journal: Splitting hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392, size=0 (0bytes) at 1733154413615Finishing writing output for hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 so closing down at 1733154417650 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000037-wal.1733154413392.temp at 1733154417672 (+22 ms)3 split writer threads finished at 1733154417673 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000037-wal.1733154413392.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733154417679 (+6 ms)Rename recovered edits hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000037-wal.1733154413392.temp to hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000066 at 1733154417680 (+1 ms)Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392, size=0, length=0, corrupted=false, cancelled=false at 1733154417680 2024-12-02T15:46:57,681 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 to hdfs://localhost:33469/hbase/oldWALs/wal.1733154413392 2024-12-02T15:46:57,682 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000066 2024-12-02T15:46:57,682 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-02T15:46:57,684 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33469/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134, archiveDir=hdfs://localhost:33469/hbase/oldWALs, maxLogs=32 2024-12-02T15:46:57,696 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154417684, exclude list is [], retry=0 2024-12-02T15:46:57,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38685,DS-6e826597-5c5b-4d40-98e7-7e5b93544faf,DISK] 2024-12-02T15:46:57,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43105,DS-1266b24c-285f-4f76-a775-e330a6410b42,DISK] 2024-12-02T15:46:57,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42367,DS-59ba2a70-8a90-4941-a55d-dab4aed6a05f,DISK] 2024-12-02T15:46:57,720 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154417684 2024-12-02T15:46:57,721 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34763:34763),(127.0.0.1/127.0.0.1:44449:44449),(127.0.0.1/127.0.0.1:33971:33971)] 2024-12-02T15:46:57,721 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T15:46:57,722 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,723 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName a 2024-12-02T15:46:57,723 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:57,727 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/a/413067cf1549490090f785324e25f92e 2024-12-02T15:46:57,727 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:57,727 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,728 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName b 2024-12-02T15:46:57,728 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:57,732 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/b/12d6f523faad4aad9bbf704e8f724196 2024-12-02T15:46:57,733 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:57,733 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,733 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d5704228f8da4dce8e669078aa110d columnFamilyName c 2024-12-02T15:46:57,733 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T15:46:57,738 DEBUG [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/c/b34be5bf3fc34e1ebd8c22f633a04f55 2024-12-02T15:46:57,738 INFO [StoreOpener-c1d5704228f8da4dce8e669078aa110d-1 {}] regionserver.HStore(327): Store=c1d5704228f8da4dce8e669078aa110d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T15:46:57,739 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,739 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,742 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000066 2024-12-02T15:46:57,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-02T15:46:57,768 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000066 2024-12-02T15:46:57,768 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1d5704228f8da4dce8e669078aa110d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-02T15:46:57,781 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/a/fdfe59fbbeac4ede8eea17d129ed7d89 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733154413542/Put/seqid=0 2024-12-02T15:46:57,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741940_1120 (size=5958) 2024-12-02T15:46:57,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741940_1120 (size=5958) 2024-12-02T15:46:57,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741940_1120 (size=5958) 2024-12-02T15:46:57,788 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/a/fdfe59fbbeac4ede8eea17d129ed7d89 2024-12-02T15:46:57,803 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/b/92e52f5408e64990b6dc17b2078ad437 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733154413551/Put/seqid=0 2024-12-02T15:46:57,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741941_1121 (size=5958) 2024-12-02T15:46:57,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741941_1121 (size=5958) 2024-12-02T15:46:57,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741941_1121 (size=5958) 2024-12-02T15:46:57,809 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/b/92e52f5408e64990b6dc17b2078ad437 2024-12-02T15:46:57,826 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/c/c09c996b9223424187d986160d651888 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733154413559/Put/seqid=0 2024-12-02T15:46:57,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741942_1122 (size=5958) 2024-12-02T15:46:57,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741942_1122 (size=5958) 2024-12-02T15:46:57,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741942_1122 (size=5958) 2024-12-02T15:46:57,831 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/c/c09c996b9223424187d986160d651888 2024-12-02T15:46:57,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/a/fdfe59fbbeac4ede8eea17d129ed7d89 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/a/fdfe59fbbeac4ede8eea17d129ed7d89 2024-12-02T15:46:57,840 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/a/fdfe59fbbeac4ede8eea17d129ed7d89, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T15:46:57,841 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/b/92e52f5408e64990b6dc17b2078ad437 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/b/92e52f5408e64990b6dc17b2078ad437 2024-12-02T15:46:57,845 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/b/92e52f5408e64990b6dc17b2078ad437, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T15:46:57,846 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/.tmp/c/c09c996b9223424187d986160d651888 as hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/c/c09c996b9223424187d986160d651888 2024-12-02T15:46:57,850 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/c/c09c996b9223424187d986160d651888, entries=10, sequenceid=66, filesize=5.8 K 2024-12-02T15:46:57,851 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c1d5704228f8da4dce8e669078aa110d in 82ms, sequenceid=66, compaction requested=false; wal=null 2024-12-02T15:46:57,851 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/0000000000000000066 2024-12-02T15:46:57,852 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,852 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,853 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T15:46:57,854 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1d5704228f8da4dce8e669078aa110d 2024-12-02T15:46:57,856 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/hbase/data/default/testReplayEditsWrittenViaHRegion/c1d5704228f8da4dce8e669078aa110d/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-02T15:46:57,857 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1d5704228f8da4dce8e669078aa110d; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68429598, jitterRate=0.01968047022819519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T15:46:57,858 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1d5704228f8da4dce8e669078aa110d: Writing region info on filesystem at 1733154417721Initializing all the Stores at 1733154417722 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154417722Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154417722Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733154417722Obtaining lock to block concurrent updates at 1733154417768 (+46 ms)Preparing flush snapshotting stores in c1d5704228f8da4dce8e669078aa110d at 1733154417769 (+1 ms)Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733154417769Flushing stores of testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. at 1733154417769Flushing c1d5704228f8da4dce8e669078aa110d/a: creating writer at 1733154417769Flushing c1d5704228f8da4dce8e669078aa110d/a: appending metadata at 1733154417780 (+11 ms)Flushing c1d5704228f8da4dce8e669078aa110d/a: closing flushed file at 1733154417780Flushing c1d5704228f8da4dce8e669078aa110d/b: creating writer at 1733154417791 (+11 ms)Flushing c1d5704228f8da4dce8e669078aa110d/b: appending metadata at 1733154417803 (+12 ms)Flushing c1d5704228f8da4dce8e669078aa110d/b: closing flushed file at 1733154417803Flushing c1d5704228f8da4dce8e669078aa110d/c: creating writer at 1733154417813 (+10 ms)Flushing c1d5704228f8da4dce8e669078aa110d/c: appending metadata at 1733154417825 (+12 ms)Flushing c1d5704228f8da4dce8e669078aa110d/c: closing flushed file at 1733154417825Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e3e092: reopening flushed file at 1733154417835 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13054e3f: reopening flushed file at 1733154417840 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@411cb7b7: reopening flushed file at 1733154417845 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c1d5704228f8da4dce8e669078aa110d in 82ms, sequenceid=66, compaction requested=false; wal=null at 1733154417851 (+6 ms)Cleaning up temporary data from old regions at 1733154417852 (+1 ms)Region opened successfully at 1733154417858 (+6 ms) 2024-12-02T15:46:57,870 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1d5704228f8da4dce8e669078aa110d, disabling compactions & flushes 2024-12-02T15:46:57,870 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:57,870 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:57,870 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. after waiting 0 ms 2024-12-02T15:46:57,870 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:57,872 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733154413135.c1d5704228f8da4dce8e669078aa110d. 2024-12-02T15:46:57,872 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1d5704228f8da4dce8e669078aa110d: Waiting for close lock at 1733154417870Disabling compacts and flushes for region at 1733154417870Disabling writes for close at 1733154417870Writing region close event to WAL at 1733154417872 (+2 ms)Closed at 1733154417872 2024-12-02T15:46:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741939_1119 (size=175) 2024-12-02T15:46:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741939_1119 (size=175) 2024-12-02T15:46:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741939_1119 (size=175) 2024-12-02T15:46:57,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-02T15:46:57,879 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733154417684) 2024-12-02T15:46:57,893 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=441 (was 437) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_696493439_22 at /127.0.0.1:50118 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_696493439_22 at /127.0.0.1:56948 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33469 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33469 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1409 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=9690 (was 9692) 2024-12-02T15:46:57,893 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1409 is superior to 1024 2024-12-02T15:46:57,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T15:46:57,893 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T15:46:57,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T15:46:57,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:57,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:57,894 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T15:46:57,894 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T15:46:57,894 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=655895052, stopped=false 2024-12-02T15:46:57,895 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e1c0b6c2739c,42629,1733154376263 2024-12-02T15:46:57,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:57,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:57,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T15:46:57,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:57,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:57,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:57,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T15:46:57,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T15:46:57,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T15:46:57,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:57,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:57,952 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1c0b6c2739c,40775,1733154377022' ***** 2024-12-02T15:46:57,953 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T15:46:57,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:57,953 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e1c0b6c2739c,39549,1733154377118' ***** 2024-12-02T15:46:57,953 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T15:46:57,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T15:46:57,953 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T15:46:57,953 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T15:46:57,953 INFO [RS:1;e1c0b6c2739c:39549 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T15:46:57,953 INFO [RS:1;e1c0b6c2739c:39549 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T15:46:57,953 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T15:46:57,953 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(959): stopping server e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T15:46:57,953 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(3091): Received CLOSE for 86be1823288224745d6875510237d50b 2024-12-02T15:46:57,953 INFO [RS:1;e1c0b6c2739c:39549 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e1c0b6c2739c:39549. 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(959): stopping server e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T15:46:57,953 DEBUG [RS:1;e1c0b6c2739c:39549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T15:46:57,953 DEBUG [RS:1;e1c0b6c2739c:39549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:57,953 INFO [RS:0;e1c0b6c2739c:40775 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e1c0b6c2739c:40775. 2024-12-02T15:46:57,954 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(976): stopping server e1c0b6c2739c,39549,1733154377118; all regions closed. 2024-12-02T15:46:57,954 DEBUG [RS:0;e1c0b6c2739c:40775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T15:46:57,953 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 86be1823288224745d6875510237d50b, disabling compactions & flushes 2024-12-02T15:46:57,954 DEBUG [RS:0;e1c0b6c2739c:40775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:57,954 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:57,954 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T15:46:57,954 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:57,954 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T15:46:57,954 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T15:46:57,954 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. after waiting 0 ms 2024-12-02T15:46:57,954 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:57,954 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T15:46:57,954 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T15:46:57,954 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 86be1823288224745d6875510237d50b=testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b.} 2024-12-02T15:46:57,954 DEBUG [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 86be1823288224745d6875510237d50b 2024-12-02T15:46:57,954 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T15:46:57,954 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T15:46:57,954 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T15:46:57,954 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T15:46:57,955 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T15:46:57,955 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-02T15:46:57,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741833_1009 (size=175) 2024-12-02T15:46:57,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741833_1009 (size=175) 2024-12-02T15:46:57,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741833_1009 (size=175) 2024-12-02T15:46:57,960 DEBUG [RS:1;e1c0b6c2739c:39549 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs 2024-12-02T15:46:57,960 INFO [RS:1;e1c0b6c2739c:39549 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1c0b6c2739c%2C39549%2C1733154377118:(num 1733154378800) 2024-12-02T15:46:57,960 DEBUG [RS:1;e1c0b6c2739c:39549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:57,960 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T15:46:57,960 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T15:46:57,961 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.ChoreService(370): Chore service for: regionserver/e1c0b6c2739c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T15:46:57,961 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T15:46:57,961 INFO [regionserver/e1c0b6c2739c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T15:46:57,961 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T15:46:57,961 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T15:46:57,961 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T15:46:57,961 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/info/5a29a686f9a0451ebf4e25b719de82aa is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b./info:regioninfo/1733154397124/Put/seqid=0 2024-12-02T15:46:57,961 INFO [RS:1;e1c0b6c2739c:39549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39549 2024-12-02T15:46:57,963 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/default/testReplayEditsAfterRegionMovedWithMultiCF/86be1823288224745d6875510237d50b/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-02T15:46:57,963 INFO [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:57,963 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 86be1823288224745d6875510237d50b: Waiting for close lock at 1733154417953Running coprocessor pre-close hooks at 1733154417953Disabling compacts and flushes for region at 1733154417953Disabling writes for close at 1733154417954 (+1 ms)Writing region close event to WAL at 1733154417959 (+5 ms)Running coprocessor post-close hooks at 1733154417963 (+4 ms)Closed at 1733154417963 2024-12-02T15:46:57,963 DEBUG [RS_CLOSE_REGION-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733154393000.86be1823288224745d6875510237d50b. 2024-12-02T15:46:57,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741943_1123 (size=6778) 2024-12-02T15:46:57,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741943_1123 (size=6778) 2024-12-02T15:46:57,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741943_1123 (size=6778) 2024-12-02T15:46:57,968 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/info/5a29a686f9a0451ebf4e25b719de82aa 2024-12-02T15:46:57,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T15:46:57,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1c0b6c2739c,39549,1733154377118 2024-12-02T15:46:57,971 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T15:46:57,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1c0b6c2739c,39549,1733154377118] 2024-12-02T15:46:57,974 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/.tmp/info/5a29a686f9a0451ebf4e25b719de82aa as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/info/5a29a686f9a0451ebf4e25b719de82aa 2024-12-02T15:46:57,979 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/info/5a29a686f9a0451ebf4e25b719de82aa, entries=8, sequenceid=23, filesize=6.6 K 2024-12-02T15:46:57,980 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false 2024-12-02T15:46:57,985 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-02T15:46:57,985 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T15:46:57,985 INFO [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T15:46:57,985 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733154417954Running coprocessor pre-close hooks at 1733154417954Disabling compacts and flushes for region at 1733154417954Disabling writes for close at 1733154417954Obtaining lock to block concurrent updates at 1733154417955 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733154417955Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733154417955Flushing stores of hbase:meta,,1.1588230740 at 1733154417956 (+1 ms)Flushing 1588230740/info: creating writer at 1733154417956Flushing 1588230740/info: appending metadata at 1733154417961 (+5 ms)Flushing 1588230740/info: closing flushed file at 1733154417961Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c4981da: reopening flushed file at 1733154417973 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false at 1733154417980 (+7 ms)Writing region close event to WAL at 1733154417982 (+2 ms)Running coprocessor post-close hooks at 1733154417985 (+3 ms)Closed at 1733154417985 2024-12-02T15:46:57,986 DEBUG [RS_CLOSE_META-regionserver/e1c0b6c2739c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T15:46:57,987 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1c0b6c2739c,39549,1733154377118 already deleted, retry=false 2024-12-02T15:46:57,988 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e1c0b6c2739c,39549,1733154377118 expired; onlineServers=1 2024-12-02T15:46:58,001 INFO [regionserver/e1c0b6c2739c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T15:46:58,001 INFO [regionserver/e1c0b6c2739c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T15:46:58,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:58,080 INFO [RS:1;e1c0b6c2739c:39549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T15:46:58,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39549-0x1019802a9bc0002, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:58,080 INFO [RS:1;e1c0b6c2739c:39549 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1c0b6c2739c,39549,1733154377118; zookeeper connection closed. 2024-12-02T15:46:58,080 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4482711d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4482711d 2024-12-02T15:46:58,154 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(976): stopping server e1c0b6c2739c,40775,1733154377022; all regions closed. 2024-12-02T15:46:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741893_1071 (size=2625) 2024-12-02T15:46:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741893_1071 (size=2625) 2024-12-02T15:46:58,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741893_1071 (size=2625) 2024-12-02T15:46:58,160 DEBUG [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs 2024-12-02T15:46:58,160 INFO [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1c0b6c2739c%2C40775%2C1733154377022.meta:.meta(num 1733154396200) 2024-12-02T15:46:58,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741835_1011 (size=3621) 2024-12-02T15:46:58,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741835_1011 (size=3621) 2024-12-02T15:46:58,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741835_1011 (size=3621) 2024-12-02T15:46:58,166 DEBUG [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/oldWALs 2024-12-02T15:46:58,166 INFO [RS:0;e1c0b6c2739c:40775 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e1c0b6c2739c%2C40775%2C1733154377022:(num 1733154378800) 2024-12-02T15:46:58,166 DEBUG [RS:0;e1c0b6c2739c:40775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T15:46:58,166 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T15:46:58,166 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T15:46:58,166 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.ChoreService(370): Chore service for: regionserver/e1c0b6c2739c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T15:46:58,167 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T15:46:58,167 INFO [regionserver/e1c0b6c2739c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T15:46:58,167 INFO [RS:0;e1c0b6c2739c:40775 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40775 2024-12-02T15:46:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e1c0b6c2739c,40775,1733154377022 2024-12-02T15:46:58,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T15:46:58,176 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T15:46:58,185 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e1c0b6c2739c,40775,1733154377022] 2024-12-02T15:46:58,193 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e1c0b6c2739c,40775,1733154377022 already deleted, retry=false 2024-12-02T15:46:58,193 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e1c0b6c2739c,40775,1733154377022 expired; onlineServers=0 2024-12-02T15:46:58,193 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e1c0b6c2739c,42629,1733154376263' ***** 2024-12-02T15:46:58,193 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T15:46:58,193 INFO [M:0;e1c0b6c2739c:42629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T15:46:58,193 INFO [M:0;e1c0b6c2739c:42629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T15:46:58,193 DEBUG [M:0;e1c0b6c2739c:42629 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T15:46:58,194 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T15:46:58,194 DEBUG [M:0;e1c0b6c2739c:42629 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T15:46:58,194 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster-HFileCleaner.small.0-1733154378406 {}] cleaner.HFileCleaner(306): Exit Thread[master/e1c0b6c2739c:0:becomeActiveMaster-HFileCleaner.small.0-1733154378406,5,FailOnTimeoutGroup] 2024-12-02T15:46:58,194 DEBUG [master/e1c0b6c2739c:0:becomeActiveMaster-HFileCleaner.large.0-1733154378405 {}] cleaner.HFileCleaner(306): Exit Thread[master/e1c0b6c2739c:0:becomeActiveMaster-HFileCleaner.large.0-1733154378405,5,FailOnTimeoutGroup] 2024-12-02T15:46:58,194 INFO [M:0;e1c0b6c2739c:42629 {}] hbase.ChoreService(370): Chore service for: master/e1c0b6c2739c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T15:46:58,194 INFO [M:0;e1c0b6c2739c:42629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T15:46:58,194 DEBUG [M:0;e1c0b6c2739c:42629 {}] master.HMaster(1795): Stopping service threads 2024-12-02T15:46:58,194 INFO [M:0;e1c0b6c2739c:42629 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T15:46:58,194 INFO [M:0;e1c0b6c2739c:42629 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T15:46:58,195 INFO [M:0;e1c0b6c2739c:42629 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T15:46:58,195 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T15:46:58,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T15:46:58,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T15:46:58,205 DEBUG [M:0;e1c0b6c2739c:42629 {}] zookeeper.ZKUtil(347): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T15:46:58,205 WARN [M:0;e1c0b6c2739c:42629 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T15:46:58,205 INFO [M:0;e1c0b6c2739c:42629 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/.lastflushedseqids 2024-12-02T15:46:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741944_1124 (size=119) 2024-12-02T15:46:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741944_1124 (size=119) 2024-12-02T15:46:58,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741944_1124 (size=119) 2024-12-02T15:46:58,219 INFO [M:0;e1c0b6c2739c:42629 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T15:46:58,219 INFO [M:0;e1c0b6c2739c:42629 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T15:46:58,219 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T15:46:58,219 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:58,219 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:58,219 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T15:46:58,219 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:58,219 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-12-02T15:46:58,233 DEBUG [M:0;e1c0b6c2739c:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eadfaf0b6f404ad2b76fc29a633ecfd1 is 82, key is hbase:meta,,1/info:regioninfo/1733154396402/Put/seqid=0 2024-12-02T15:46:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741945_1125 (size=6063) 2024-12-02T15:46:58,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741945_1125 (size=6063) 2024-12-02T15:46:58,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741945_1125 (size=6063) 2024-12-02T15:46:58,239 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eadfaf0b6f404ad2b76fc29a633ecfd1 2024-12-02T15:46:58,262 DEBUG [M:0;e1c0b6c2739c:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d25165f555364de4856e731a3e3a9ba1 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733154393425/Put/seqid=0 2024-12-02T15:46:58,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741946_1126 (size=7907) 2024-12-02T15:46:58,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741946_1126 (size=7907) 2024-12-02T15:46:58,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741946_1126 (size=7907) 2024-12-02T15:46:58,268 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d25165f555364de4856e731a3e3a9ba1 2024-12-02T15:46:58,272 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d25165f555364de4856e731a3e3a9ba1 2024-12-02T15:46:58,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:58,285 INFO [RS:0;e1c0b6c2739c:40775 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T15:46:58,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40775-0x1019802a9bc0001, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:58,285 INFO [RS:0;e1c0b6c2739c:40775 {}] regionserver.HRegionServer(1031): Exiting; stopping=e1c0b6c2739c,40775,1733154377022; zookeeper connection closed. 2024-12-02T15:46:58,285 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@179204b9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@179204b9 2024-12-02T15:46:58,285 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-02T15:46:58,286 DEBUG [M:0;e1c0b6c2739c:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a7d7763e6fa48619d640800080f1466 is 69, key is e1c0b6c2739c,39549,1733154377118/rs:state/1733154378452/Put/seqid=0 2024-12-02T15:46:58,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741947_1127 (size=5440) 2024-12-02T15:46:58,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741947_1127 (size=5440) 2024-12-02T15:46:58,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741947_1127 (size=5440) 2024-12-02T15:46:58,294 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a7d7763e6fa48619d640800080f1466 2024-12-02T15:46:58,298 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2a7d7763e6fa48619d640800080f1466 2024-12-02T15:46:58,299 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eadfaf0b6f404ad2b76fc29a633ecfd1 as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eadfaf0b6f404ad2b76fc29a633ecfd1 2024-12-02T15:46:58,305 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eadfaf0b6f404ad2b76fc29a633ecfd1, entries=14, sequenceid=207, filesize=5.9 K 2024-12-02T15:46:58,306 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d25165f555364de4856e731a3e3a9ba1 as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d25165f555364de4856e731a3e3a9ba1 2024-12-02T15:46:58,310 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d25165f555364de4856e731a3e3a9ba1 2024-12-02T15:46:58,310 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d25165f555364de4856e731a3e3a9ba1, entries=21, sequenceid=207, filesize=7.7 K 2024-12-02T15:46:58,311 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a7d7763e6fa48619d640800080f1466 as hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a7d7763e6fa48619d640800080f1466 2024-12-02T15:46:58,315 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2a7d7763e6fa48619d640800080f1466 2024-12-02T15:46:58,315 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33469/user/jenkins/test-data/32849a6c-01fd-cad0-eedc-07875d0ffa82/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a7d7763e6fa48619d640800080f1466, entries=3, sequenceid=207, filesize=5.3 K 2024-12-02T15:46:58,316 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false 2024-12-02T15:46:58,317 INFO [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T15:46:58,317 DEBUG [M:0;e1c0b6c2739c:42629 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733154418219Disabling compacts and flushes for region at 1733154418219Disabling writes for close at 1733154418219Obtaining lock to block concurrent updates at 1733154418219Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733154418219Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1733154418220 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733154418220Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733154418220Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733154418233 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733154418233Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733154418243 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733154418261 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733154418261Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733154418273 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733154418285 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733154418285Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@466a42a6: reopening flushed file at 1733154418298 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14bc1132: reopening flushed file at 1733154418305 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59cf7033: reopening flushed file at 1733154418310 (+5 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=207, compaction requested=false at 1733154418316 (+6 ms)Writing region close event to WAL at 1733154418317 (+1 ms)Closed at 1733154418317 2024-12-02T15:46:58,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43105 is added to blk_1073741830_1006 (size=102162) 2024-12-02T15:46:58,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42367 is added to blk_1073741830_1006 (size=102162) 2024-12-02T15:46:58,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38685 is added to blk_1073741830_1006 (size=102162) 2024-12-02T15:46:58,320 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T15:46:58,320 INFO [M:0;e1c0b6c2739c:42629 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T15:46:58,320 INFO [M:0;e1c0b6c2739c:42629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42629 2024-12-02T15:46:58,321 INFO [M:0;e1c0b6c2739c:42629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T15:46:58,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:58,430 INFO [M:0;e1c0b6c2739c:42629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T15:46:58,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x1019802a9bc0000, quorum=127.0.0.1:53141, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T15:46:58,436 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154407506 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154407506 (inode 16688) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733154398171/wal.1733154407506 (inode 16688) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-02T15:46:58,440 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397978 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397978 (inode 16665) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733154397611/wal.1733154397978 (inode 16665) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-02T15:46:58,440 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733154388516/wal.1733154388628 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:58,442 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154408427 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154408427 (inode 16714) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733154407763/wal.1733154408427 (inode 16714) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-02T15:46:58,443 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733154408603/wal.1733154408679 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:58,443 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733154413134/wal.1733154413392 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:58,443 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733154381366/wal.1733154381455 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T15:46:58,446 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal.1733154381197 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal.1733154381197 (inode 16485) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733154380424/wal.1733154381197 (inode 16485) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-02T15:46:58,449 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal.1733154380153 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal.1733154380153 (inode 16462) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733154379727/wal.1733154380153 (inode 16462) Holder DFSClient_NONMAPREDUCE_1260188680_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-02T15:46:58,484 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T15:46:58,486 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T15:46:58,486 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T15:46:58,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T15:46:58,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,STOPPED} 2024-12-02T15:46:58,489 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T15:46:58,489 WARN [BP-1466660644-172.17.0.3-1733154372547 heartbeating to localhost/127.0.0.1:33469 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T15:46:58,489 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T15:46:58,489 WARN [BP-1466660644-172.17.0.3-1733154372547 heartbeating to localhost/127.0.0.1:33469 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1466660644-172.17.0.3-1733154372547 (Datanode Uuid 9cf4a451-2c25-42fd-8fe4-b2b5063f94ec) service to localhost/127.0.0.1:33469 2024-12-02T15:46:58,491 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data5/current/BP-1466660644-172.17.0.3-1733154372547 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T15:46:58,492 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data6/current/BP-1466660644-172.17.0.3-1733154372547 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T15:46:58,493 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T15:46:58,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T15:46:58,496 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T15:46:58,496 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T15:46:58,496 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T15:46:58,497 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,STOPPED} 2024-12-02T15:46:58,498 WARN [BP-1466660644-172.17.0.3-1733154372547 heartbeating to localhost/127.0.0.1:33469 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T15:46:58,498 WARN [BP-1466660644-172.17.0.3-1733154372547 heartbeating to localhost/127.0.0.1:33469 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1466660644-172.17.0.3-1733154372547 (Datanode Uuid 51b79570-c33f-49df-95ef-06df9bdb5ba3) service to localhost/127.0.0.1:33469 2024-12-02T15:46:58,498 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T15:46:58,498 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T15:46:58,498 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data3/current/BP-1466660644-172.17.0.3-1733154372547 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T15:46:58,499 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data4/current/BP-1466660644-172.17.0.3-1733154372547 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T15:46:58,499 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T15:46:58,507 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T15:46:58,508 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T15:46:58,508 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T15:46:58,508 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T15:46:58,508 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,STOPPED} 2024-12-02T15:46:58,509 WARN [BP-1466660644-172.17.0.3-1733154372547 heartbeating to localhost/127.0.0.1:33469 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T15:46:58,509 WARN [BP-1466660644-172.17.0.3-1733154372547 heartbeating to localhost/127.0.0.1:33469 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1466660644-172.17.0.3-1733154372547 (Datanode Uuid c236a47e-8e13-4fb3-8647-23ed49cac4a3) service to localhost/127.0.0.1:33469 2024-12-02T15:46:58,510 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data1/current/BP-1466660644-172.17.0.3-1733154372547 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T15:46:58,510 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/cluster_52df4aca-7c23-fe5b-3712-c3058c50a658/data/data2/current/BP-1466660644-172.17.0.3-1733154372547 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T15:46:58,511 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T15:46:58,516 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T15:46:58,516 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T15:46:58,519 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T15:46:58,520 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T15:46:58,520 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T15:46:58,520 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T15:46:58,521 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd226707-9c91-4308-e8c4-0c8218017c6c/hadoop.log.dir/,STOPPED} 2024-12-02T15:46:58,532 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T15:46:58,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down